2024-04-14 11:15:15,014 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@394a2528 2024-04-14 11:15:15,027 main DEBUG Took 0.010720 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-04-14 11:15:15,027 main DEBUG PluginManager 'Core' found 129 plugins 2024-04-14 11:15:15,028 main DEBUG PluginManager 'Level' found 0 plugins 2024-04-14 11:15:15,029 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-04-14 11:15:15,030 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:15:15,038 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-04-14 11:15:15,050 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:15:15,052 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:15:15,052 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:15:15,053 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:15:15,053 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:15:15,053 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:15:15,054 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:15:15,054 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:15:15,055 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:15:15,055 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:15:15,055 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:15:15,056 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:15:15,056 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:15:15,056 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:15:15,057 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:15:15,057 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:15:15,057 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:15:15,058 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:15:15,058 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:15:15,059 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:15:15,059 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:15:15,059 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:15:15,060 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:15:15,060 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:15:15,061 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:15:15,061 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-04-14 11:15:15,064 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:15:15,066 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-04-14 11:15:15,068 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-04-14 11:15:15,069 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-04-14 11:15:15,070 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-04-14 11:15:15,070 main DEBUG PluginManager 'Converter' found 47 plugins 2024-04-14 11:15:15,082 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-04-14 11:15:15,084 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-04-14 11:15:15,085 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-04-14 11:15:15,086 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-04-14 11:15:15,086 main DEBUG createAppenders(={Console}) 2024-04-14 11:15:15,087 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@394a2528 initialized 2024-04-14 11:15:15,087 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@394a2528 2024-04-14 11:15:15,087 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@394a2528 OK. 2024-04-14 11:15:15,088 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-04-14 11:15:15,088 main DEBUG OutputStream closed 2024-04-14 11:15:15,088 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-04-14 11:15:15,089 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-04-14 11:15:15,089 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5e0e82ae OK 2024-04-14 11:15:15,135 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2024-04-14 11:15:15,137 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2024-04-14 11:15:15,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2024-04-14 11:15:15,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2024-04-14 11:15:15,140 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2024-04-14 11:15:15,140 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-04-14 11:15:15,140 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2024-04-14 11:15:15,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-04-14 11:15:15,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-04-14 11:15:15,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-04-14 11:15:15,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2024-04-14 11:15:15,142 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2024-04-14 11:15:15,142 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-04-14 11:15:15,142 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-04-14 11:15:15,143 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-04-14 11:15:15,143 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-04-14 11:15:15,143 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-04-14 11:15:15,144 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2024-04-14 11:15:15,147 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2024-04-14 11:15:15,148 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6221a451) with optional ClassLoader: null 2024-04-14 11:15:15,148 main DEBUG Shutdown hook enabled. Registering a new one. 2024-04-14 11:15:15,149 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@6221a451] started OK. 2024-04-14T11:15:15,394 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4 2024-04-14 11:15:15,398 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-04-14 11:15:15,399 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2024-04-14T11:15:15,410 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-04-14T11:15:15,423 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-04-14T11:15:15,427 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/cluster_4b1b29d7-47f8-fcef-3a29-5ae20b7f3813, deleteOnExit=true 2024-04-14T11:15:15,427 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-04-14T11:15:15,428 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/test.cache.data in system properties and HBase conf 2024-04-14T11:15:15,429 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/hadoop.tmp.dir in system properties and HBase conf 2024-04-14T11:15:15,429 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/hadoop.log.dir in system properties and HBase conf 2024-04-14T11:15:15,430 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/mapreduce.cluster.local.dir in system properties and HBase conf 2024-04-14T11:15:15,431 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-04-14T11:15:15,431 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-04-14T11:15:15,553 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-04-14T11:15:15,994 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-14T11:15:15,999 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-04-14T11:15:15,999 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-04-14T11:15:16,000 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-04-14T11:15:16,000 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-04-14T11:15:16,000 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-04-14T11:15:16,001 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-04-14T11:15:16,001 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-04-14T11:15:16,001 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-04-14T11:15:16,002 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-04-14T11:15:16,002 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/nfs.dump.dir in system properties and HBase conf 2024-04-14T11:15:16,002 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/java.io.tmpdir in system properties and HBase conf 2024-04-14T11:15:16,003 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-04-14T11:15:16,003 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-04-14T11:15:16,003 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2024-04-14T11:15:16,498 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2024-04-14T11:15:16,502 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2024-04-14T11:15:16,795 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-04-14T11:15:16,923 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@69161862 via org.mortbay.log.Slf4jLog 2024-04-14T11:15:16,939 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2024-04-14T11:15:16,978 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2024-04-14T11:15:17,015 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/java.io.tmpdir/Jetty_localhost_40329_hdfs____z0xvfs/webapp 2024-04-14T11:15:17,147 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40329 2024-04-14T11:15:17,167 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2024-04-14T11:15:17,168 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2024-04-14T11:15:17,728 WARN [Listener at localhost/40697 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2024-04-14T11:15:17,791 WARN [Listener at localhost/40697 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2024-04-14T11:15:17,811 WARN [Listener at localhost/40697 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2024-04-14T11:15:17,818 INFO [Listener at localhost/40697 {}] log.Slf4jLog(67): jetty-6.1.26 2024-04-14T11:15:17,824 INFO [Listener at localhost/40697 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/java.io.tmpdir/Jetty_localhost_46669_datanode____i8h6ou/webapp 2024-04-14T11:15:17,927 INFO [Listener at localhost/40697 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:46669 2024-04-14T11:15:18,161 WARN [Listener at localhost/34025 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2024-04-14T11:15:18,462 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2024-04-14T11:15:18,641 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x15a2a5c36cebefc: Processing first storage report for DS-69693a39-a002-4a69-b452-4144d2dde82a from datanode 5f4d4e3a-51b2-4c7c-b5ce-4641021b8767 2024-04-14T11:15:18,642 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x15a2a5c36cebefc: from storage DS-69693a39-a002-4a69-b452-4144d2dde82a node DatanodeRegistration(127.0.0.1:37175, datanodeUuid=5f4d4e3a-51b2-4c7c-b5ce-4641021b8767, infoPort=43019, infoSecurePort=0, ipcPort=34025, storageInfo=lv=-57;cid=testClusterID;nsid=1068438417;c=1713093316581), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-04-14T11:15:18,643 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x15a2a5c36cebefc: Processing first storage report for DS-adcf473f-e581-4012-8f2d-61f206c330e0 from datanode 5f4d4e3a-51b2-4c7c-b5ce-4641021b8767 2024-04-14T11:15:18,643 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x15a2a5c36cebefc: from storage DS-adcf473f-e581-4012-8f2d-61f206c330e0 node DatanodeRegistration(127.0.0.1:37175, datanodeUuid=5f4d4e3a-51b2-4c7c-b5ce-4641021b8767, infoPort=43019, infoSecurePort=0, ipcPort=34025, storageInfo=lv=-57;cid=testClusterID;nsid=1068438417;c=1713093316581), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-04-14T11:15:18,722 DEBUG [Listener at localhost/34025 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4 2024-04-14T11:15:18,840 INFO [Listener at localhost/34025 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/cluster_4b1b29d7-47f8-fcef-3a29-5ae20b7f3813/zookeeper_0, clientPort=60913, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/cluster_4b1b29d7-47f8-fcef-3a29-5ae20b7f3813/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/cluster_4b1b29d7-47f8-fcef-3a29-5ae20b7f3813/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-04-14T11:15:18,858 INFO [Listener at localhost/34025 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=60913 2024-04-14T11:15:18,865 INFO [Listener at localhost/34025 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-04-14T11:15:18,867 INFO [Listener at localhost/34025 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-04-14T11:15:19,537 INFO [Listener at localhost/34025 {}] util.FSUtils(462): Created version file at hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a with version=8 2024-04-14T11:15:19,538 INFO [Listener at localhost/34025 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/hbase-staging 2024-04-14T11:15:19,748 DEBUG [Listener at localhost/34025 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-04-14T11:15:20,411 INFO [Listener at localhost/34025 {}] client.ConnectionUtils(129): master/jenkins-hbase6:0 server-side Connection retries=45 2024-04-14T11:15:20,447 INFO [Listener at localhost/34025 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-04-14T11:15:20,448 INFO [Listener at localhost/34025 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-04-14T11:15:20,448 INFO [Listener at localhost/34025 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-04-14T11:15:20,448 INFO [Listener at localhost/34025 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-04-14T11:15:20,448 INFO [Listener at localhost/34025 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-04-14T11:15:20,613 INFO [Listener at localhost/34025 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-04-14T11:15:20,712 INFO [Listener at localhost/34025 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-04-14T11:15:20,729 DEBUG [Listener at localhost/34025 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-04-14T11:15:20,736 INFO [Listener at localhost/34025 {}] ipc.NettyRpcServer(316): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-04-14T11:15:20,787 DEBUG [Listener at localhost/34025 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 3349 (auto-detected) 2024-04-14T11:15:20,788 DEBUG [Listener at localhost/34025 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:5a:ff:fe:dc:8e:c9 (auto-detected) 2024-04-14T11:15:20,832 INFO [Listener at localhost/34025 {}] ipc.NettyRpcServer(198): Bind to /172.31.2.20:33819 2024-04-14T11:15:20,844 INFO [Listener at localhost/34025 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-04-14T11:15:20,847 INFO [Listener at localhost/34025 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-04-14T11:15:20,863 INFO [Listener at localhost/34025 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:33819 connecting to ZooKeeper ensemble=127.0.0.1:60913 2024-04-14T11:15:20,908 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:338190x0, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-04-14T11:15:20,912 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:33819-0x1004d666d980000 connected 2024-04-14T11:15:20,946 DEBUG [Listener at localhost/34025 {}] zookeeper.ZKUtil(113): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-04-14T11:15:20,949 DEBUG [Listener at localhost/34025 {}] zookeeper.ZKUtil(113): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-04-14T11:15:20,953 DEBUG [Listener at localhost/34025 {}] zookeeper.ZKUtil(113): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-04-14T11:15:20,960 DEBUG [Listener at localhost/34025 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33819 2024-04-14T11:15:20,961 DEBUG [Listener at localhost/34025 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33819 2024-04-14T11:15:20,961 DEBUG [Listener at localhost/34025 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33819 2024-04-14T11:15:20,961 DEBUG [Listener at localhost/34025 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33819 2024-04-14T11:15:20,962 DEBUG [Listener at localhost/34025 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33819 2024-04-14T11:15:20,972 INFO [Listener at localhost/34025 {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a, hbase.cluster.distributed=false 2024-04-14T11:15:21,028 INFO [Listener at localhost/34025 {}] client.ConnectionUtils(129): regionserver/jenkins-hbase6:0 server-side Connection retries=45 2024-04-14T11:15:21,028 INFO [Listener at localhost/34025 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-04-14T11:15:21,028 INFO [Listener at localhost/34025 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-04-14T11:15:21,028 INFO [Listener at localhost/34025 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-04-14T11:15:21,029 INFO [Listener at localhost/34025 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-04-14T11:15:21,029 INFO [Listener at localhost/34025 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-04-14T11:15:21,034 INFO [Listener at localhost/34025 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-04-14T11:15:21,036 INFO [Listener at localhost/34025 {}] ipc.NettyRpcServer(316): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-04-14T11:15:21,037 INFO [Listener at localhost/34025 {}] ipc.NettyRpcServer(198): Bind to /172.31.2.20:33239 2024-04-14T11:15:21,039 INFO [Listener at localhost/34025 {}] hfile.BlockCacheFactory(116): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2024-04-14T11:15:21,045 DEBUG [Listener at localhost/34025 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-04-14T11:15:21,046 INFO [Listener at localhost/34025 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-04-14T11:15:21,049 INFO [Listener at localhost/34025 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-04-14T11:15:21,051 INFO [Listener at localhost/34025 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:33239 connecting to ZooKeeper ensemble=127.0.0.1:60913 2024-04-14T11:15:21,055 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): regionserver:332390x0, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-04-14T11:15:21,056 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:33239-0x1004d666d980001 connected 2024-04-14T11:15:21,056 DEBUG [Listener at localhost/34025 {}] zookeeper.ZKUtil(113): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-04-14T11:15:21,058 DEBUG [Listener at localhost/34025 {}] zookeeper.ZKUtil(113): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-04-14T11:15:21,059 DEBUG [Listener at localhost/34025 {}] zookeeper.ZKUtil(113): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-04-14T11:15:21,060 DEBUG [Listener at localhost/34025 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33239 2024-04-14T11:15:21,060 DEBUG [Listener at localhost/34025 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33239 2024-04-14T11:15:21,060 DEBUG [Listener at localhost/34025 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33239 2024-04-14T11:15:21,061 DEBUG [Listener at localhost/34025 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33239 2024-04-14T11:15:21,061 DEBUG [Listener at localhost/34025 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33239 2024-04-14T11:15:21,064 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(2434): Adding backup master ZNode /hbase/backup-masters/jenkins-hbase6.apache.org,33819,1713093319730 2024-04-14T11:15:21,076 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-04-14T11:15:21,076 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-04-14T11:15:21,079 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/jenkins-hbase6.apache.org,33819,1713093319730 2024-04-14T11:15:21,092 DEBUG [M:0;jenkins-hbase6:33819 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase6:33819 2024-04-14T11:15:21,102 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-04-14T11:15:21,102 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-04-14T11:15:21,102 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-04-14T11:15:21,102 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-04-14T11:15:21,103 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-04-14T11:15:21,105 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/jenkins-hbase6.apache.org,33819,1713093319730 from backup master directory 2024-04-14T11:15:21,106 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-04-14T11:15:21,109 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/jenkins-hbase6.apache.org,33819,1713093319730 2024-04-14T11:15:21,109 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-04-14T11:15:21,109 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-04-14T11:15:21,110 WARN [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-04-14T11:15:21,110 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase6.apache.org,33819,1713093319730 2024-04-14T11:15:21,113 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2024-04-14T11:15:21,114 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2024-04-14T11:15:21,195 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/hbase.id with ID: 3c034648-392a-4f08-9849-c36bc7ee5a9e 2024-04-14T11:15:21,236 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-04-14T11:15:21,251 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-04-14T11:15:21,251 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-04-14T11:15:21,292 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-04-14T11:15:21,295 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-04-14T11:15:21,313 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2024-04-14T11:15:21,313 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2024-04-14T11:15:21,314 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:146) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2459) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:21,323 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2459) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:21,325 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-04-14T11:15:21,798 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7106): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store 2024-04-14T11:15:21,822 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-04-14T11:15:21,823 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(891): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:15:21,826 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1676): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-04-14T11:15:21,826 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1698): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-04-14T11:15:21,826 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1719): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-04-14T11:15:21,826 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1786): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-04-14T11:15:21,827 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1796): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-04-14T11:15:21,827 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1917): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-04-14T11:15:21,827 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1630): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-04-14T11:15:21,829 WARN [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store/.initializing 2024-04-14T11:15:21,829 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/WALs/jenkins-hbase6.apache.org,33819,1713093319730 2024-04-14T11:15:21,840 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-04-14T11:15:21,854 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.AbstractFSWAL(498): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase6.apache.org%2C33819%2C1713093319730, suffix=, logDir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/WALs/jenkins-hbase6.apache.org,33819,1713093319730, archiveDir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/oldWALs, maxLogs=10 2024-04-14T11:15:21,892 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/WALs/jenkins-hbase6.apache.org,33819,1713093319730/jenkins-hbase6.apache.org%2C33819%2C1713093319730.1713093321861, exclude list is [], retry=0 2024-04-14T11:15:21,924 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37175,DS-69693a39-a002-4a69-b452-4144d2dde82a,DISK] 2024-04-14T11:15:21,933 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:654) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:499) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:407) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.6.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.6.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:22,006 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.AbstractFSWAL(839): New WAL /user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/WALs/jenkins-hbase6.apache.org,33819,1713093319730/jenkins-hbase6.apache.org%2C33819%2C1713093319730.1713093321861 2024-04-14T11:15:22,007 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37175,DS-69693a39-a002-4a69-b452-4144d2dde82a,DISK]] 2024-04-14T11:15:22,007 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7267): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-04-14T11:15:22,008 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(891): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:15:22,012 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7309): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-04-14T11:15:22,014 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7312): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-04-14T11:15:22,077 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-04-14T11:15:22,109 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-04-14T11:15:22,114 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:22,118 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-04-14T11:15:22,118 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-04-14T11:15:22,122 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-04-14T11:15:22,123 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:22,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:15:22,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-04-14T11:15:22,128 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-04-14T11:15:22,128 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:22,130 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:15:22,130 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-04-14T11:15:22,133 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-04-14T11:15:22,133 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:22,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:15:22,138 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-04-14T11:15:22,140 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-04-14T11:15:22,153 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-04-14T11:15:22,157 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1082): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-04-14T11:15:22,162 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-04-14T11:15:22,163 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1099): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59942023, jitterRate=-0.10679425299167633}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-04-14T11:15:22,169 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(998): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-04-14T11:15:22,171 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-04-14T11:15:22,220 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68f2c3d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:22,272 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-04-14T11:15:22,292 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-04-14T11:15:22,292 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(611): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-04-14T11:15:22,295 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-04-14T11:15:22,297 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(631): Recovered RegionProcedureStore lease in 1 msec 2024-04-14T11:15:22,303 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(645): Loaded RegionProcedureStore in 6 msec 2024-04-14T11:15:22,304 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-04-14T11:15:22,341 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-04-14T11:15:22,359 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-04-14T11:15:22,362 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/balancer already deleted, retry=false 2024-04-14T11:15:22,365 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-04-14T11:15:22,367 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-04-14T11:15:22,369 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/normalizer already deleted, retry=false 2024-04-14T11:15:22,372 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-04-14T11:15:22,377 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-04-14T11:15:22,379 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/switch/split already deleted, retry=false 2024-04-14T11:15:22,380 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-04-14T11:15:22,382 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/switch/merge already deleted, retry=false 2024-04-14T11:15:22,396 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-04-14T11:15:22,397 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-04-14T11:15:22,409 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-04-14T11:15:22,409 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-04-14T11:15:22,409 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-04-14T11:15:22,409 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-04-14T11:15:22,410 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=jenkins-hbase6.apache.org,33819,1713093319730, sessionid=0x1004d666d980000, setting cluster-up flag (Was=false) 2024-04-14T11:15:22,427 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-04-14T11:15:22,427 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-04-14T11:15:22,434 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-04-14T11:15:22,436 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase6.apache.org,33819,1713093319730 2024-04-14T11:15:22,441 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-04-14T11:15:22,441 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-04-14T11:15:22,448 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-04-14T11:15:22,450 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase6.apache.org,33819,1713093319730 2024-04-14T11:15:22,476 DEBUG [RS:0;jenkins-hbase6:33239 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase6:33239 2024-04-14T11:15:22,478 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(1005): ClusterId : 3c034648-392a-4f08-9849-c36bc7ee5a9e 2024-04-14T11:15:22,481 DEBUG [RS:0;jenkins-hbase6:33239 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-04-14T11:15:22,487 DEBUG [RS:0;jenkins-hbase6:33239 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-04-14T11:15:22,487 DEBUG [RS:0;jenkins-hbase6:33239 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-04-14T11:15:22,492 DEBUG [RS:0;jenkins-hbase6:33239 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-04-14T11:15:22,493 DEBUG [RS:0;jenkins-hbase6:33239 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@182e2aab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:22,496 DEBUG [RS:0;jenkins-hbase6:33239 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c842e09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=jenkins-hbase6.apache.org/172.31.2.20:0 2024-04-14T11:15:22,503 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-04-14T11:15:22,503 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-04-14T11:15:22,503 DEBUG [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(1087): About to register with Master. 2024-04-14T11:15:22,507 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(3064): reportForDuty to master=jenkins-hbase6.apache.org,33819,1713093319730 with isa=jenkins-hbase6.apache.org/172.31.2.20:33239, startcode=1713093321027 2024-04-14T11:15:22,533 DEBUG [RS:0;jenkins-hbase6:33239 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-04-14T11:15:22,587 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1080): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-04-14T11:15:22,595 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-04-14T11:15:22,597 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:39507, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-04-14T11:15:22,599 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-04-14T11:15:22,605 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33819 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3268) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:592) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17346) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:22,614 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase6.apache.org,33819,1713093319730 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-04-14T11:15:22,619 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2024-04-14T11:15:22,620 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2024-04-14T11:15:22,620 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2024-04-14T11:15:22,620 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2024-04-14T11:15:22,620 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase6:0, corePoolSize=10, maxPoolSize=10 2024-04-14T11:15:22,621 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2024-04-14T11:15:22,621 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=2, maxPoolSize=2 2024-04-14T11:15:22,621 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2024-04-14T11:15:22,622 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1713093352622 2024-04-14T11:15:22,624 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-04-14T11:15:22,633 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-04-14T11:15:22,634 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-04-14T11:15:22,639 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:22,639 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-04-14T11:15:22,640 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-04-14T11:15:22,654 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-04-14T11:15:22,654 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-04-14T11:15:22,655 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-04-14T11:15:22,655 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-04-14T11:15:22,656 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-04-14T11:15:22,656 INFO [PEWorker-1 {}] regionserver.HRegion(7088): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a 2024-04-14T11:15:22,656 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-04-14T11:15:22,658 DEBUG [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(3088): Master is not running yet 2024-04-14T11:15:22,658 WARN [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(1096): reportForDuty failed; sleeping 100 ms and then retrying. 2024-04-14T11:15:22,661 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-04-14T11:15:22,663 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-04-14T11:15:22,664 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-04-14T11:15:22,671 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-04-14T11:15:22,671 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-04-14T11:15:22,674 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.large.0-1713093322673,5,FailOnTimeoutGroup] 2024-04-14T11:15:22,674 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.small.0-1713093322674,5,FailOnTimeoutGroup] 2024-04-14T11:15:22,674 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-04-14T11:15:22,675 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-04-14T11:15:22,676 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-04-14T11:15:22,676 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-04-14T11:15:22,682 DEBUG [PEWorker-1 {}] regionserver.HRegion(891): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:15:22,685 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-04-14T11:15:22,688 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-04-14T11:15:22,688 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:22,689 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-04-14T11:15:22,690 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-04-14T11:15:22,692 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-04-14T11:15:22,692 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:22,693 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-04-14T11:15:22,693 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-04-14T11:15:22,695 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-04-14T11:15:22,696 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:22,696 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-04-14T11:15:22,699 DEBUG [PEWorker-1 {}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740 2024-04-14T11:15:22,699 DEBUG [PEWorker-1 {}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740 2024-04-14T11:15:22,703 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-04-14T11:15:22,705 DEBUG [PEWorker-1 {}] regionserver.HRegion(1082): writing seq id for 1588230740 2024-04-14T11:15:22,709 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-04-14T11:15:22,710 INFO [PEWorker-1 {}] regionserver.HRegion(1099): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59966625, jitterRate=-0.10642765462398529}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-04-14T11:15:22,713 DEBUG [PEWorker-1 {}] regionserver.HRegion(998): Region open journal for 1588230740: 2024-04-14T11:15:22,713 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Closing 1588230740, disabling compactions & flushes 2024-04-14T11:15:22,713 INFO [PEWorker-1 {}] regionserver.HRegion(1698): Closing region hbase:meta,,1.1588230740 2024-04-14T11:15:22,713 DEBUG [PEWorker-1 {}] regionserver.HRegion(1719): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-04-14T11:15:22,713 DEBUG [PEWorker-1 {}] regionserver.HRegion(1786): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-04-14T11:15:22,713 DEBUG [PEWorker-1 {}] regionserver.HRegion(1796): Updates disabled for region hbase:meta,,1.1588230740 2024-04-14T11:15:22,715 INFO [PEWorker-1 {}] regionserver.HRegion(1917): Closed hbase:meta,,1.1588230740 2024-04-14T11:15:22,715 DEBUG [PEWorker-1 {}] regionserver.HRegion(1630): Region close journal for 1588230740: 2024-04-14T11:15:22,719 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-04-14T11:15:22,719 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-04-14T11:15:22,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-04-14T11:15:22,739 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(729): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-04-14T11:15:22,743 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-04-14T11:15:22,759 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(3064): reportForDuty to master=jenkins-hbase6.apache.org,33819,1713093319730 with isa=jenkins-hbase6.apache.org/172.31.2.20:33239, startcode=1713093321027 2024-04-14T11:15:22,761 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33819 {}] master.ServerManager(332): Checking decommissioned status of RegionServer jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:22,766 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33819 {}] master.ServerManager(486): Registering regionserver=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:22,778 DEBUG [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(1722): Config from master: hbase.rootdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a 2024-04-14T11:15:22,778 DEBUG [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(1722): Config from master: fs.defaultFS=hdfs://localhost:40697 2024-04-14T11:15:22,778 DEBUG [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(1722): Config from master: hbase.master.info.port=-1 2024-04-14T11:15:22,784 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-04-14T11:15:22,785 DEBUG [RS:0;jenkins-hbase6:33239 {}] zookeeper.ZKUtil(111): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:22,785 WARN [RS:0;jenkins-hbase6:33239 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-04-14T11:15:22,785 INFO [RS:0;jenkins-hbase6:33239 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-04-14T11:15:22,785 DEBUG [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(2097): logDir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/WALs/jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:22,788 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase6.apache.org,33239,1713093321027] 2024-04-14T11:15:22,803 DEBUG [RS:0;jenkins-hbase6:33239 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-04-14T11:15:22,818 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-04-14T11:15:22,833 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2024-04-14T11:15:22,837 INFO [RS:0;jenkins-hbase6:33239 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-04-14T11:15:22,838 INFO [RS:0;jenkins-hbase6:33239 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-04-14T11:15:22,839 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer$CompactionChecker(1985): CompactionChecker runs every PT1S 2024-04-14T11:15:22,847 INFO [RS:0;jenkins-hbase6:33239 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-04-14T11:15:22,848 DEBUG [RS:0;jenkins-hbase6:33239 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2024-04-14T11:15:22,848 DEBUG [RS:0;jenkins-hbase6:33239 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2024-04-14T11:15:22,848 DEBUG [RS:0;jenkins-hbase6:33239 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2024-04-14T11:15:22,848 DEBUG [RS:0;jenkins-hbase6:33239 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2024-04-14T11:15:22,848 DEBUG [RS:0;jenkins-hbase6:33239 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2024-04-14T11:15:22,848 DEBUG [RS:0;jenkins-hbase6:33239 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase6:0, corePoolSize=2, maxPoolSize=2 2024-04-14T11:15:22,848 DEBUG [RS:0;jenkins-hbase6:33239 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2024-04-14T11:15:22,848 DEBUG [RS:0;jenkins-hbase6:33239 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2024-04-14T11:15:22,849 DEBUG [RS:0;jenkins-hbase6:33239 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2024-04-14T11:15:22,849 DEBUG [RS:0;jenkins-hbase6:33239 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2024-04-14T11:15:22,849 DEBUG [RS:0;jenkins-hbase6:33239 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2024-04-14T11:15:22,849 DEBUG [RS:0;jenkins-hbase6:33239 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0, corePoolSize=3, maxPoolSize=3 2024-04-14T11:15:22,849 DEBUG [RS:0;jenkins-hbase6:33239 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0, corePoolSize=3, maxPoolSize=3 2024-04-14T11:15:22,850 INFO [RS:0;jenkins-hbase6:33239 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-04-14T11:15:22,850 INFO [RS:0;jenkins-hbase6:33239 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-04-14T11:15:22,850 INFO [RS:0;jenkins-hbase6:33239 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-04-14T11:15:22,851 INFO [RS:0;jenkins-hbase6:33239 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-04-14T11:15:22,851 INFO [RS:0;jenkins-hbase6:33239 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,33239,1713093321027-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-04-14T11:15:22,867 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-04-14T11:15:22,869 INFO [RS:0;jenkins-hbase6:33239 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,33239,1713093321027-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-04-14T11:15:22,886 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.Replication(204): jenkins-hbase6.apache.org,33239,1713093321027 started 2024-04-14T11:15:22,886 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(1764): Serving as jenkins-hbase6.apache.org,33239,1713093321027, RpcServer on jenkins-hbase6.apache.org/172.31.2.20:33239, sessionid=0x1004d666d980001 2024-04-14T11:15:22,887 DEBUG [RS:0;jenkins-hbase6:33239 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-04-14T11:15:22,887 DEBUG [RS:0;jenkins-hbase6:33239 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:22,887 DEBUG [RS:0;jenkins-hbase6:33239 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase6.apache.org,33239,1713093321027' 2024-04-14T11:15:22,887 DEBUG [RS:0;jenkins-hbase6:33239 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-04-14T11:15:22,888 DEBUG [RS:0;jenkins-hbase6:33239 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-04-14T11:15:22,889 DEBUG [RS:0;jenkins-hbase6:33239 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-04-14T11:15:22,889 DEBUG [RS:0;jenkins-hbase6:33239 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-04-14T11:15:22,889 DEBUG [RS:0;jenkins-hbase6:33239 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:22,889 DEBUG [RS:0;jenkins-hbase6:33239 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase6.apache.org,33239,1713093321027' 2024-04-14T11:15:22,889 DEBUG [RS:0;jenkins-hbase6:33239 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-04-14T11:15:22,890 DEBUG [RS:0;jenkins-hbase6:33239 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-04-14T11:15:22,891 DEBUG [RS:0;jenkins-hbase6:33239 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-04-14T11:15:22,891 INFO [RS:0;jenkins-hbase6:33239 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-04-14T11:15:22,891 INFO [RS:0;jenkins-hbase6:33239 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-04-14T11:15:22,894 WARN [jenkins-hbase6:33819 {}] assignment.AssignmentManager(2340): No servers available; cannot place 1 unassigned regions. 2024-04-14T11:15:23,000 INFO [RS:0;jenkins-hbase6:33239 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-04-14T11:15:23,003 INFO [RS:0;jenkins-hbase6:33239 {}] wal.AbstractFSWAL(498): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase6.apache.org%2C33239%2C1713093321027, suffix=, logDir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/WALs/jenkins-hbase6.apache.org,33239,1713093321027, archiveDir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/oldWALs, maxLogs=32 2024-04-14T11:15:23,017 DEBUG [RS:0;jenkins-hbase6:33239 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/WALs/jenkins-hbase6.apache.org,33239,1713093321027/jenkins-hbase6.apache.org%2C33239%2C1713093321027.1713093323006, exclude list is [], retry=0 2024-04-14T11:15:23,024 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37175,DS-69693a39-a002-4a69-b452-4144d2dde82a,DISK] 2024-04-14T11:15:23,029 INFO [RS:0;jenkins-hbase6:33239 {}] wal.AbstractFSWAL(839): New WAL /user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/WALs/jenkins-hbase6.apache.org,33239,1713093321027/jenkins-hbase6.apache.org%2C33239%2C1713093321027.1713093323006 2024-04-14T11:15:23,029 DEBUG [RS:0;jenkins-hbase6:33239 {}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37175,DS-69693a39-a002-4a69-b452-4144d2dde82a,DISK]] 2024-04-14T11:15:23,146 DEBUG [jenkins-hbase6:33819 {}] assignment.AssignmentManager(2361): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-04-14T11:15:23,154 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:23,161 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase6.apache.org,33239,1713093321027, state=OPENING 2024-04-14T11:15:23,168 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-04-14T11:15:23,170 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-04-14T11:15:23,170 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-04-14T11:15:23,171 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-04-14T11:15:23,171 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-04-14T11:15:23,174 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:15:23,361 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:23,363 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-04-14T11:15:23,369 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:55534, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-04-14T11:15:23,383 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2024-04-14T11:15:23,384 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-04-14T11:15:23,384 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-04-14T11:15:23,388 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(498): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase6.apache.org%2C33239%2C1713093321027.meta, suffix=.meta, logDir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/WALs/jenkins-hbase6.apache.org,33239,1713093321027, archiveDir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/oldWALs, maxLogs=32 2024-04-14T11:15:23,401 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(536): When create output stream for /user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/WALs/jenkins-hbase6.apache.org,33239,1713093321027/jenkins-hbase6.apache.org%2C33239%2C1713093321027.meta.1713093323389.meta, exclude list is [], retry=0 2024-04-14T11:15:23,407 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37175,DS-69693a39-a002-4a69-b452-4144d2dde82a,DISK] 2024-04-14T11:15:23,412 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(839): New WAL /user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/WALs/jenkins-hbase6.apache.org,33239,1713093321027/jenkins-hbase6.apache.org%2C33239%2C1713093321027.meta.1713093323389.meta 2024-04-14T11:15:23,412 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(923): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:37175,DS-69693a39-a002-4a69-b452-4144d2dde82a,DISK]] 2024-04-14T11:15:23,413 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7267): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-04-14T11:15:23,414 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-04-14T11:15:23,432 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7981): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-04-14T11:15:23,438 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-04-14T11:15:23,444 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-04-14T11:15:23,444 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(891): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:15:23,445 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7309): checking encryption for 1588230740 2024-04-14T11:15:23,445 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7312): checking classloading for 1588230740 2024-04-14T11:15:23,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-04-14T11:15:23,450 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-04-14T11:15:23,451 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:23,451 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-04-14T11:15:23,452 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-04-14T11:15:23,453 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-04-14T11:15:23,453 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:23,454 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-04-14T11:15:23,454 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-04-14T11:15:23,455 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-04-14T11:15:23,456 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:23,456 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-04-14T11:15:23,458 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740 2024-04-14T11:15:23,461 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740 2024-04-14T11:15:23,464 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-04-14T11:15:23,467 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1082): writing seq id for 1588230740 2024-04-14T11:15:23,469 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1099): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62409270, jitterRate=-0.07002940773963928}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-04-14T11:15:23,471 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(998): Region open journal for 1588230740: 2024-04-14T11:15:23,482 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2593): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1713093323352 2024-04-14T11:15:23,501 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2620): Finished post open deploy task for hbase:meta,,1.1588230740 2024-04-14T11:15:23,502 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2024-04-14T11:15:23,503 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:23,505 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase6.apache.org,33239,1713093321027, state=OPEN 2024-04-14T11:15:23,513 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-04-14T11:15:23,513 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-04-14T11:15:23,513 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-04-14T11:15:23,513 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-04-14T11:15:23,518 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=3, resume processing ppid=2 2024-04-14T11:15:23,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase6.apache.org,33239,1713093321027 in 339 msec 2024-04-14T11:15:23,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=2, resume processing ppid=1 2024-04-14T11:15:23,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 792 msec 2024-04-14T11:15:23,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.0230 sec 2024-04-14T11:15:23,533 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1713093323533, completionTime=-1 2024-04-14T11:15:23,534 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-04-14T11:15:23,534 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] assignment.AssignmentManager(1664): Joining cluster... 2024-04-14T11:15:23,599 DEBUG [hconnection-0x7a2db419-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:23,603 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:55550, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:23,618 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] assignment.AssignmentManager(1676): Number of RegionServers=1 2024-04-14T11:15:23,619 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1713093383619 2024-04-14T11:15:23,619 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1713093443619 2024-04-14T11:15:23,619 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] assignment.AssignmentManager(1683): Joined the cluster in 85 msec 2024-04-14T11:15:23,645 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,33819,1713093319730-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-04-14T11:15:23,646 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,33819,1713093319730-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-04-14T11:15:23,646 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,33819,1713093319730-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-04-14T11:15:23,648 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase6:33819, period=300000, unit=MILLISECONDS is enabled. 2024-04-14T11:15:23,648 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-04-14T11:15:23,658 DEBUG [master/jenkins-hbase6:0.Chore.1 {}] janitor.CatalogJanitor(175): 2024-04-14T11:15:23,664 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-04-14T11:15:23,666 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(2414): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-04-14T11:15:23,675 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1080): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-04-14T11:15:23,680 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(84): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-04-14T11:15:23,681 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:23,683 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(84): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-04-14T11:15:24,106 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7088): creating {ENCODED => 23980ec0107c2c55bc841b5f2502d0a1, NAME => 'hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a 2024-04-14T11:15:24,119 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(891): Instantiated hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:15:24,119 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1676): Closing 23980ec0107c2c55bc841b5f2502d0a1, disabling compactions & flushes 2024-04-14T11:15:24,119 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1698): Closing region hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1. 2024-04-14T11:15:24,119 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1719): Waiting without time limit for close lock on hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1. 2024-04-14T11:15:24,120 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1786): Acquired close lock on hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1. after waiting 0 ms 2024-04-14T11:15:24,120 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1796): Updates disabled for region hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1. 2024-04-14T11:15:24,120 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1917): Closed hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1. 2024-04-14T11:15:24,120 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1630): Region close journal for 23980ec0107c2c55bc841b5f2502d0a1: 2024-04-14T11:15:24,122 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(84): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-04-14T11:15:24,137 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1713093324124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1713093324124"}]},"ts":"1713093324124"} 2024-04-14T11:15:24,171 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-04-14T11:15:24,174 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(84): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-04-14T11:15:24,179 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093324174"}]},"ts":"1713093324174"} 2024-04-14T11:15:24,186 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-04-14T11:15:24,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=23980ec0107c2c55bc841b5f2502d0a1, ASSIGN}] 2024-04-14T11:15:24,197 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(729): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=23980ec0107c2c55bc841b5f2502d0a1, ASSIGN 2024-04-14T11:15:24,199 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=23980ec0107c2c55bc841b5f2502d0a1, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,33239,1713093321027; forceNewPlan=false, retain=false 2024-04-14T11:15:24,350 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=23980ec0107c2c55bc841b5f2502d0a1, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:24,356 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 23980ec0107c2c55bc841b5f2502d0a1, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:15:24,510 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:24,518 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1. 2024-04-14T11:15:24,518 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7267): Opening region: {ENCODED => 23980ec0107c2c55bc841b5f2502d0a1, NAME => 'hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1.', STARTKEY => '', ENDKEY => ''} 2024-04-14T11:15:24,519 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 23980ec0107c2c55bc841b5f2502d0a1 2024-04-14T11:15:24,519 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(891): Instantiated hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:15:24,519 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7309): checking encryption for 23980ec0107c2c55bc841b5f2502d0a1 2024-04-14T11:15:24,519 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7312): checking classloading for 23980ec0107c2c55bc841b5f2502d0a1 2024-04-14T11:15:24,522 INFO [StoreOpener-23980ec0107c2c55bc841b5f2502d0a1-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 23980ec0107c2c55bc841b5f2502d0a1 2024-04-14T11:15:24,525 INFO [StoreOpener-23980ec0107c2c55bc841b5f2502d0a1-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23980ec0107c2c55bc841b5f2502d0a1 columnFamilyName info 2024-04-14T11:15:24,525 DEBUG [StoreOpener-23980ec0107c2c55bc841b5f2502d0a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:24,526 INFO [StoreOpener-23980ec0107c2c55bc841b5f2502d0a1-1 {}] regionserver.HStore(324): Store=23980ec0107c2c55bc841b5f2502d0a1/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:15:24,529 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/namespace/23980ec0107c2c55bc841b5f2502d0a1 2024-04-14T11:15:24,530 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/namespace/23980ec0107c2c55bc841b5f2502d0a1 2024-04-14T11:15:24,534 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1082): writing seq id for 23980ec0107c2c55bc841b5f2502d0a1 2024-04-14T11:15:24,539 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/namespace/23980ec0107c2c55bc841b5f2502d0a1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-04-14T11:15:24,540 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1099): Opened 23980ec0107c2c55bc841b5f2502d0a1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58945894, jitterRate=-0.1216377317905426}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-04-14T11:15:24,541 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(998): Region open journal for 23980ec0107c2c55bc841b5f2502d0a1: 2024-04-14T11:15:24,543 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2593): Post open deploy tasks for hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1., pid=6, masterSystemTime=1713093324510 2024-04-14T11:15:24,548 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2620): Finished post open deploy task for hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1. 2024-04-14T11:15:24,548 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1. 2024-04-14T11:15:24,549 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=23980ec0107c2c55bc841b5f2502d0a1, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:24,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=6, resume processing ppid=5 2024-04-14T11:15:24,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 23980ec0107c2c55bc841b5f2502d0a1, server=jenkins-hbase6.apache.org,33239,1713093321027 in 200 msec 2024-04-14T11:15:24,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=5, resume processing ppid=4 2024-04-14T11:15:24,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=23980ec0107c2c55bc841b5f2502d0a1, ASSIGN in 365 msec 2024-04-14T11:15:24,565 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(84): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-04-14T11:15:24,565 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093324565"}]},"ts":"1713093324565"} 2024-04-14T11:15:24,569 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-04-14T11:15:24,573 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(84): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-04-14T11:15:24,576 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 906 msec 2024-04-14T11:15:24,580 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-04-14T11:15:24,582 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-04-14T11:15:24,582 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-04-14T11:15:24,582 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-04-14T11:15:24,632 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1080): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-04-14T11:15:24,657 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-04-14T11:15:24,666 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 40 msec 2024-04-14T11:15:24,681 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1080): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-04-14T11:15:24,698 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-04-14T11:15:24,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 23 msec 2024-04-14T11:15:24,722 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-04-14T11:15:24,725 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-04-14T11:15:24,725 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 3.615sec 2024-04-14T11:15:24,728 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-04-14T11:15:24,730 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-04-14T11:15:24,731 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-04-14T11:15:24,732 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-04-14T11:15:24,732 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-04-14T11:15:24,733 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,33819,1713093319730-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-04-14T11:15:24,734 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,33819,1713093319730-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-04-14T11:15:24,747 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-04-14T11:15:24,748 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-04-14T11:15:24,749 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,33819,1713093319730-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-04-14T11:15:24,779 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x302b29d8 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:24,779 WARN [Listener at localhost/34025 {}] client.ZKConnectionRegistry(83): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-04-14T11:15:24,789 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@549778db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:24,794 DEBUG [Listener at localhost/34025 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-04-14T11:15:24,794 DEBUG [Listener at localhost/34025 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-04-14T11:15:24,809 DEBUG [hconnection-0x63792811-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:24,823 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:55566, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:24,838 INFO [Listener at localhost/34025 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase6.apache.org,33819,1713093319730 2024-04-14T11:15:24,880 INFO [Listener at localhost/34025 {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=205, OpenFileDescriptor=506, MaxFileDescriptor=60000, SystemLoadAverage=316, ProcessCount=168, AvailableMemoryMB=2885 2024-04-14T11:15:24,898 DEBUG [Listener at localhost/34025 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-04-14T11:15:24,901 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:36614, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-04-14T11:15:24,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-04-14T11:15:24,917 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$4(2378): Client=jenkins//172.31.2.20 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-04-14T11:15:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-04-14T11:15:24,922 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(84): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-04-14T11:15:24,923 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:24,924 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 9 2024-04-14T11:15:24,924 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(84): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-04-14T11:15:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2024-04-14T11:15:24,945 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7088): creating {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a 2024-04-14T11:15:24,961 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(891): Instantiated TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:15:24,962 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Closing 5407acadb5e9c9af78221a8f2d7e6785, disabling compactions & flushes 2024-04-14T11:15:24,962 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1698): Closing region TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:24,962 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1719): Waiting without time limit for close lock on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:24,962 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1786): Acquired close lock on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. after waiting 0 ms 2024-04-14T11:15:24,962 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1796): Updates disabled for region TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:24,962 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1917): Closed TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:24,962 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1630): Region close journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:24,964 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(84): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-04-14T11:15:24,965 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1713093324964"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1713093324964"}]},"ts":"1713093324964"} 2024-04-14T11:15:24,968 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-04-14T11:15:24,969 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(84): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-04-14T11:15:24,969 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093324969"}]},"ts":"1713093324969"} 2024-04-14T11:15:24,972 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-04-14T11:15:24,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=5407acadb5e9c9af78221a8f2d7e6785, ASSIGN}] 2024-04-14T11:15:24,979 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(729): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=5407acadb5e9c9af78221a8f2d7e6785, ASSIGN 2024-04-14T11:15:24,980 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5407acadb5e9c9af78221a8f2d7e6785, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,33239,1713093321027; forceNewPlan=false, retain=false 2024-04-14T11:15:25,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2024-04-14T11:15:25,132 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=5407acadb5e9c9af78221a8f2d7e6785, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:25,136 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure 5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:15:25,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2024-04-14T11:15:25,290 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:25,297 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:25,298 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7267): Opening region: {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} 2024-04-14T11:15:25,298 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:25,298 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(891): Instantiated TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:15:25,298 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7309): checking encryption for 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:25,298 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7312): checking classloading for 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:25,301 INFO [StoreOpener-5407acadb5e9c9af78221a8f2d7e6785-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:25,304 INFO [StoreOpener-5407acadb5e9c9af78221a8f2d7e6785-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5407acadb5e9c9af78221a8f2d7e6785 columnFamilyName A 2024-04-14T11:15:25,304 DEBUG [StoreOpener-5407acadb5e9c9af78221a8f2d7e6785-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:25,305 INFO [StoreOpener-5407acadb5e9c9af78221a8f2d7e6785-1 {}] regionserver.HStore(324): Store=5407acadb5e9c9af78221a8f2d7e6785/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:15:25,305 INFO [StoreOpener-5407acadb5e9c9af78221a8f2d7e6785-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:25,307 INFO [StoreOpener-5407acadb5e9c9af78221a8f2d7e6785-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5407acadb5e9c9af78221a8f2d7e6785 columnFamilyName B 2024-04-14T11:15:25,307 DEBUG [StoreOpener-5407acadb5e9c9af78221a8f2d7e6785-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:25,308 INFO [StoreOpener-5407acadb5e9c9af78221a8f2d7e6785-1 {}] regionserver.HStore(324): Store=5407acadb5e9c9af78221a8f2d7e6785/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:15:25,308 INFO [StoreOpener-5407acadb5e9c9af78221a8f2d7e6785-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:25,310 INFO [StoreOpener-5407acadb5e9c9af78221a8f2d7e6785-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5407acadb5e9c9af78221a8f2d7e6785 columnFamilyName C 2024-04-14T11:15:25,310 DEBUG [StoreOpener-5407acadb5e9c9af78221a8f2d7e6785-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:25,311 INFO [StoreOpener-5407acadb5e9c9af78221a8f2d7e6785-1 {}] regionserver.HStore(324): Store=5407acadb5e9c9af78221a8f2d7e6785/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:15:25,313 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:25,314 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:25,317 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-04-14T11:15:25,319 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1082): writing seq id for 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:25,324 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-04-14T11:15:25,325 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1099): Opened 5407acadb5e9c9af78221a8f2d7e6785; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74026064, jitterRate=0.10307431221008301}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-04-14T11:15:25,327 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(998): Region open journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:25,328 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2593): Post open deploy tasks for TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., pid=11, masterSystemTime=1713093325290 2024-04-14T11:15:25,332 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2620): Finished post open deploy task for TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:25,332 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:25,334 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=5407acadb5e9c9af78221a8f2d7e6785, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:25,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=11, resume processing ppid=10 2024-04-14T11:15:25,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure 5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 in 202 msec 2024-04-14T11:15:25,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=10, resume processing ppid=9 2024-04-14T11:15:25,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=5407acadb5e9c9af78221a8f2d7e6785, ASSIGN in 364 msec 2024-04-14T11:15:25,347 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(84): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-04-14T11:15:25,347 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093325347"}]},"ts":"1713093325347"} 2024-04-14T11:15:25,350 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-04-14T11:15:25,355 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(84): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-04-14T11:15:25,358 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 438 msec 2024-04-14T11:15:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2024-04-14T11:15:25,552 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 9 completed 2024-04-14T11:15:25,559 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7cb13939 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:25,564 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c5f8b6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:25,568 DEBUG [Listener at localhost/34025 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:25,571 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:55572, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:25,574 DEBUG [Listener at localhost/34025 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-04-14T11:15:25,577 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:36620, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-04-14T11:15:25,587 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2d8ad5a3 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:25,594 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1ba55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:25,595 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3310b2d1 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:25,601 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10175683, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:25,602 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x72f2ac7d to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:25,609 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5862be58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:25,611 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x263d3c32 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:25,616 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c961bc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:25,618 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6b82d154 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:25,624 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75397465, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:25,626 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x036ba6f6 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:25,631 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@189f0fdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:25,632 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x008f2bdd to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:25,637 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e8d612c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:25,639 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1d7aa563 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:25,643 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28cb23e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:25,645 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x05800936 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:25,650 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d19e558, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:25,657 DEBUG [hconnection-0x5746b398-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:25,658 DEBUG [hconnection-0x607e4470-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:25,660 DEBUG [hconnection-0x6cf74b9a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:25,662 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:55578, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:25,662 DEBUG [hconnection-0x556fbc25-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:25,665 DEBUG [hconnection-0x1858fc47-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:25,666 DEBUG [hconnection-0x58cc7009-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:25,667 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:55604, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:25,668 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:55590, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:25,668 DEBUG [hconnection-0x5c1a63a3-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:25,672 DEBUG [hconnection-0x3c24eaef-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:25,675 DEBUG [hconnection-0xe81d6e5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:25,678 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:55608, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:25,681 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:55616, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:25,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:15:25,687 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:55618, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:25,695 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:55626, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees 2024-04-14T11:15:25,698 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:55630, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2024-04-14T11:15:25,704 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:15:25,707 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=12, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:15:25,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:15:25,719 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:55636, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:25,775 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:25,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-04-14T11:15:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2024-04-14T11:15:25,889 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:25,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-04-14T11:15:25,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/d3ef8418b61b4c70bb0bd681a9533a52 is 50, key is test_row_0/A:col10/1713093325758/Put/seqid=0 2024-04-14T11:15:25,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:25,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:25,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:25,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:25,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:25,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:25,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/d3ef8418b61b4c70bb0bd681a9533a52 2024-04-14T11:15:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2024-04-14T11:15:26,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093385981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093385991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093385992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093385998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093386010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,102 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-04-14T11:15:26,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:26,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:26,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:26,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:26,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:26,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093386122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/e98a4f4f27f94c429bc7c23966ad7630 is 50, key is test_row_0/B:col10/1713093325758/Put/seqid=0 2024-04-14T11:15:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093386123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093386126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093386127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093386128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/e98a4f4f27f94c429bc7c23966ad7630 2024-04-14T11:15:26,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/ba3418be93184e94931f2e43daad8b12 is 50, key is test_row_0/C:col10/1713093325758/Put/seqid=0 2024-04-14T11:15:26,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/ba3418be93184e94931f2e43daad8b12 2024-04-14T11:15:26,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/d3ef8418b61b4c70bb0bd681a9533a52 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d3ef8418b61b4c70bb0bd681a9533a52 2024-04-14T11:15:26,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d3ef8418b61b4c70bb0bd681a9533a52, entries=150, sequenceid=13, filesize=11.7 K 2024-04-14T11:15:26,274 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/e98a4f4f27f94c429bc7c23966ad7630 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/e98a4f4f27f94c429bc7c23966ad7630 2024-04-14T11:15:26,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-04-14T11:15:26,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:26,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:26,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:26,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:26,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:26,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/e98a4f4f27f94c429bc7c23966ad7630, entries=150, sequenceid=13, filesize=11.7 K 2024-04-14T11:15:26,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/ba3418be93184e94931f2e43daad8b12 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/ba3418be93184e94931f2e43daad8b12 2024-04-14T11:15:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2024-04-14T11:15:26,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093386330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093386332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093386338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093386339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/ba3418be93184e94931f2e43daad8b12, entries=150, sequenceid=13, filesize=11.7 K 2024-04-14T11:15:26,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5407acadb5e9c9af78221a8f2d7e6785 in 575ms, sequenceid=13, compaction requested=false 2024-04-14T11:15:26,353 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-04-14T11:15:26,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:26,366 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:26,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-04-14T11:15:26,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/d6d43aeebc7a4f6193bd0e29e133c96d is 50, key is test_row_0/A:col10/1713093326358/Put/seqid=0 2024-04-14T11:15:26,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/d6d43aeebc7a4f6193bd0e29e133c96d 2024-04-14T11:15:26,439 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/33a2fc35a4d041fe9ef7799aba56cd44 is 50, key is test_row_0/B:col10/1713093326358/Put/seqid=0 2024-04-14T11:15:26,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-04-14T11:15:26,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:26,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:26,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:26,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:26,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:26,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093386457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093386563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,599 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-04-14T11:15:26,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:26,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:26,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:26,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:26,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:26,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093386637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093386639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093386644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093386652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,757 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-04-14T11:15:26,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:26,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:26,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:26,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:26,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:26,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093386770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2024-04-14T11:15:26,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/33a2fc35a4d041fe9ef7799aba56cd44 2024-04-14T11:15:26,914 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:26,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-04-14T11:15:26,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:26,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:26,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:26,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:26,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:26,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/9a5da16d52b74a6a9868518794e59e82 is 50, key is test_row_0/C:col10/1713093326358/Put/seqid=0 2024-04-14T11:15:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:27,070 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:27,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-04-14T11:15:27,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:27,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:27,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:27,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:27,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:27,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:27,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093387077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:27,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093387141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:27,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093387145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:27,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093387152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:27,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093387164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:27,240 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:27,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-04-14T11:15:27,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:27,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:27,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:27,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:27,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:27,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/9a5da16d52b74a6a9868518794e59e82 2024-04-14T11:15:27,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/d6d43aeebc7a4f6193bd0e29e133c96d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d6d43aeebc7a4f6193bd0e29e133c96d 2024-04-14T11:15:27,396 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:27,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-04-14T11:15:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:27,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:27,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(53): pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:27,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:27,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d6d43aeebc7a4f6193bd0e29e133c96d, entries=200, sequenceid=39, filesize=14.0 K 2024-04-14T11:15:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:27,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/33a2fc35a4d041fe9ef7799aba56cd44 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/33a2fc35a4d041fe9ef7799aba56cd44 2024-04-14T11:15:27,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/33a2fc35a4d041fe9ef7799aba56cd44, entries=150, sequenceid=39, filesize=11.7 K 2024-04-14T11:15:27,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/9a5da16d52b74a6a9868518794e59e82 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/9a5da16d52b74a6a9868518794e59e82 2024-04-14T11:15:27,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/9a5da16d52b74a6a9868518794e59e82, entries=150, sequenceid=39, filesize=11.7 K 2024-04-14T11:15:27,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 5407acadb5e9c9af78221a8f2d7e6785 in 1079ms, sequenceid=39, compaction requested=false 2024-04-14T11:15:27,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:27,561 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:27,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-04-14T11:15:27,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:27,563 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-04-14T11:15:27,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/9d684e784c0a477b950805be221f3d67 is 50, key is test_row_1/A:col10/1713093326452/Put/seqid=0 2024-04-14T11:15:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:27,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:27,625 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/9d684e784c0a477b950805be221f3d67 2024-04-14T11:15:27,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/d7d55f3f334a4a6fb55ef754f2f5cb91 is 50, key is test_row_1/B:col10/1713093326452/Put/seqid=0 2024-04-14T11:15:27,681 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/d7d55f3f334a4a6fb55ef754f2f5cb91 2024-04-14T11:15:27,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/116f47cf8d894c6aa7a8cecadd4fef48 is 50, key is test_row_1/C:col10/1713093326452/Put/seqid=0 2024-04-14T11:15:27,741 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/116f47cf8d894c6aa7a8cecadd4fef48 2024-04-14T11:15:27,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/9d684e784c0a477b950805be221f3d67 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/9d684e784c0a477b950805be221f3d67 2024-04-14T11:15:27,771 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/9d684e784c0a477b950805be221f3d67, entries=100, sequenceid=49, filesize=9.4 K 2024-04-14T11:15:27,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/d7d55f3f334a4a6fb55ef754f2f5cb91 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/d7d55f3f334a4a6fb55ef754f2f5cb91 2024-04-14T11:15:27,789 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/d7d55f3f334a4a6fb55ef754f2f5cb91, entries=100, sequenceid=49, filesize=9.4 K 2024-04-14T11:15:27,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/116f47cf8d894c6aa7a8cecadd4fef48 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/116f47cf8d894c6aa7a8cecadd4fef48 2024-04-14T11:15:27,812 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/116f47cf8d894c6aa7a8cecadd4fef48, entries=100, sequenceid=49, filesize=9.4 K 2024-04-14T11:15:27,817 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(3035): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=140.89 KB/144270 for 5407acadb5e9c9af78221a8f2d7e6785 in 251ms, sequenceid=49, compaction requested=true 2024-04-14T11:15:27,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:27,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:27,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=13 2024-04-14T11:15:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=13 2024-04-14T11:15:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2024-04-14T11:15:27,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=13, resume processing ppid=12 2024-04-14T11:15:27,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=13, ppid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1070 sec 2024-04-14T11:15:27,830 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:27,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-04-14T11:15:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:27,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees in 2.1450 sec 2024-04-14T11:15:27,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/ec5c84e30cc949c298749ae2ff1e7388 is 50, key is test_row_0/A:col10/1713093327802/Put/seqid=0 2024-04-14T11:15:27,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/ec5c84e30cc949c298749ae2ff1e7388 2024-04-14T11:15:27,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093387894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:27,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/6a684de6aad542b0a1cb4b84b19889a1 is 50, key is test_row_0/B:col10/1713093327802/Put/seqid=0 2024-04-14T11:15:27,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/6a684de6aad542b0a1cb4b84b19889a1 2024-04-14T11:15:27,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/e7b7edc5c03e4d59b14e5e000b0701e2 is 50, key is test_row_0/C:col10/1713093327802/Put/seqid=0 2024-04-14T11:15:28,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093388004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/e7b7edc5c03e4d59b14e5e000b0701e2 2024-04-14T11:15:28,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/ec5c84e30cc949c298749ae2ff1e7388 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/ec5c84e30cc949c298749ae2ff1e7388 2024-04-14T11:15:28,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/ec5c84e30cc949c298749ae2ff1e7388, entries=200, sequenceid=74, filesize=14.0 K 2024-04-14T11:15:28,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/6a684de6aad542b0a1cb4b84b19889a1 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/6a684de6aad542b0a1cb4b84b19889a1 2024-04-14T11:15:28,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/6a684de6aad542b0a1cb4b84b19889a1, entries=150, sequenceid=74, filesize=11.7 K 2024-04-14T11:15:28,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/e7b7edc5c03e4d59b14e5e000b0701e2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/e7b7edc5c03e4d59b14e5e000b0701e2 2024-04-14T11:15:28,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/e7b7edc5c03e4d59b14e5e000b0701e2, entries=150, sequenceid=74, filesize=11.7 K 2024-04-14T11:15:28,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5407acadb5e9c9af78221a8f2d7e6785 in 261ms, sequenceid=74, compaction requested=true 2024-04-14T11:15:28,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:28,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:15:28,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:28,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:15:28,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:28,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:15:28,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:28,095 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:15:28,095 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:15:28,106 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:15:28,110 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/B is initiating minor compaction (all files) 2024-04-14T11:15:28,110 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/B in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:28,110 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/e98a4f4f27f94c429bc7c23966ad7630, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/33a2fc35a4d041fe9ef7799aba56cd44, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/d7d55f3f334a4a6fb55ef754f2f5cb91, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/6a684de6aad542b0a1cb4b84b19889a1] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=44.5 K 2024-04-14T11:15:28,111 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50272 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:15:28,111 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/A is initiating minor compaction (all files) 2024-04-14T11:15:28,111 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/A in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:28,112 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d3ef8418b61b4c70bb0bd681a9533a52, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d6d43aeebc7a4f6193bd0e29e133c96d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/9d684e784c0a477b950805be221f3d67, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/ec5c84e30cc949c298749ae2ff1e7388] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=49.1 K 2024-04-14T11:15:28,113 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting e98a4f4f27f94c429bc7c23966ad7630, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1713093325753 2024-04-14T11:15:28,113 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting d3ef8418b61b4c70bb0bd681a9533a52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1713093325753 2024-04-14T11:15:28,115 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting d6d43aeebc7a4f6193bd0e29e133c96d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1713093325938 2024-04-14T11:15:28,116 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 33a2fc35a4d041fe9ef7799aba56cd44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1713093325938 2024-04-14T11:15:28,117 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9d684e784c0a477b950805be221f3d67, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1713093326434 2024-04-14T11:15:28,118 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting d7d55f3f334a4a6fb55ef754f2f5cb91, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1713093326434 2024-04-14T11:15:28,119 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 6a684de6aad542b0a1cb4b84b19889a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1713093327770 2024-04-14T11:15:28,119 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting ec5c84e30cc949c298749ae2ff1e7388, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1713093327770 2024-04-14T11:15:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:28,152 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:28,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-04-14T11:15:28,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/2471ce883f8845beae4910af3241582f is 50, key is test_row_0/A:col10/1713093327889/Put/seqid=0 2024-04-14T11:15:28,193 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#B#compaction#13 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:28,194 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/106ad7c3b715403681b187833e591a3b is 50, key is test_row_0/B:col10/1713093327802/Put/seqid=0 2024-04-14T11:15:28,195 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#A#compaction#14 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:28,198 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/fca166abc2584f37b60e30b8a030cf0f is 50, key is test_row_0/A:col10/1713093327802/Put/seqid=0 2024-04-14T11:15:28,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093388250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093388249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093388253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093388253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093388254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093388362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093388362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093388364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093388364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093388366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/2471ce883f8845beae4910af3241582f 2024-04-14T11:15:28,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093388661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093388663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093388663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093388666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093388666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,707 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/fca166abc2584f37b60e30b8a030cf0f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/fca166abc2584f37b60e30b8a030cf0f 2024-04-14T11:15:28,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/c338873574c34a5dbf153b6ba866287d is 50, key is test_row_0/B:col10/1713093327889/Put/seqid=0 2024-04-14T11:15:28,712 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/106ad7c3b715403681b187833e591a3b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/106ad7c3b715403681b187833e591a3b 2024-04-14T11:15:28,746 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/A of 5407acadb5e9c9af78221a8f2d7e6785 into fca166abc2584f37b60e30b8a030cf0f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:28,746 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/B of 5407acadb5e9c9af78221a8f2d7e6785 into 106ad7c3b715403681b187833e591a3b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:28,746 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:28,746 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:28,746 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/A, priority=12, startTime=1713093328094; duration=0sec 2024-04-14T11:15:28,746 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/B, priority=12, startTime=1713093328095; duration=0sec 2024-04-14T11:15:28,747 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:28,747 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:A 2024-04-14T11:15:28,747 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:28,747 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:B 2024-04-14T11:15:28,747 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:15:28,761 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:15:28,761 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/C is initiating minor compaction (all files) 2024-04-14T11:15:28,762 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/C in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:28,762 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/ba3418be93184e94931f2e43daad8b12, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/9a5da16d52b74a6a9868518794e59e82, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/116f47cf8d894c6aa7a8cecadd4fef48, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/e7b7edc5c03e4d59b14e5e000b0701e2] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=44.5 K 2024-04-14T11:15:28,763 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting ba3418be93184e94931f2e43daad8b12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1713093325753 2024-04-14T11:15:28,764 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9a5da16d52b74a6a9868518794e59e82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1713093325938 2024-04-14T11:15:28,765 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 116f47cf8d894c6aa7a8cecadd4fef48, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1713093326434 2024-04-14T11:15:28,766 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting e7b7edc5c03e4d59b14e5e000b0701e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1713093327770 2024-04-14T11:15:28,790 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#C#compaction#16 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:28,791 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/f8952897c30b47ebb31cd6815503c07c is 50, key is test_row_0/C:col10/1713093327802/Put/seqid=0 2024-04-14T11:15:28,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093388972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093388978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093388979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093388986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:28,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093388992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:29,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/c338873574c34a5dbf153b6ba866287d 2024-04-14T11:15:29,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/953d1ddb0c644b88befffa1b900e6ecf is 50, key is test_row_0/C:col10/1713093327889/Put/seqid=0 2024-04-14T11:15:29,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/953d1ddb0c644b88befffa1b900e6ecf 2024-04-14T11:15:29,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/2471ce883f8845beae4910af3241582f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/2471ce883f8845beae4910af3241582f 2024-04-14T11:15:29,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/2471ce883f8845beae4910af3241582f, entries=200, sequenceid=86, filesize=14.0 K 2024-04-14T11:15:29,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/c338873574c34a5dbf153b6ba866287d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c338873574c34a5dbf153b6ba866287d 2024-04-14T11:15:29,238 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/f8952897c30b47ebb31cd6815503c07c as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/f8952897c30b47ebb31cd6815503c07c 2024-04-14T11:15:29,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c338873574c34a5dbf153b6ba866287d, entries=150, sequenceid=86, filesize=11.7 K 2024-04-14T11:15:29,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/953d1ddb0c644b88befffa1b900e6ecf as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/953d1ddb0c644b88befffa1b900e6ecf 2024-04-14T11:15:29,254 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/C of 5407acadb5e9c9af78221a8f2d7e6785 into f8952897c30b47ebb31cd6815503c07c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:29,256 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:29,257 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/C, priority=12, startTime=1713093328095; duration=0sec 2024-04-14T11:15:29,257 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:29,257 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:C 2024-04-14T11:15:29,258 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-04-14T11:15:29,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/953d1ddb0c644b88befffa1b900e6ecf, entries=150, sequenceid=86, filesize=11.7 K 2024-04-14T11:15:29,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5407acadb5e9c9af78221a8f2d7e6785 in 1113ms, sequenceid=86, compaction requested=false 2024-04-14T11:15:29,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:29,315 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-04-14T11:15:29,316 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-04-14T11:15:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:29,482 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:29,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-04-14T11:15:29,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/eaac03084b5240b788c680caeebf0929 is 50, key is test_row_0/A:col10/1713093328233/Put/seqid=0 2024-04-14T11:15:29,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093389508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:29,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093389509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:29,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:29,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093389513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093389513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:29,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093389514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:29,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/eaac03084b5240b788c680caeebf0929 2024-04-14T11:15:29,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/799fdcf2fc07401880bae6e97adc0f5e is 50, key is test_row_0/B:col10/1713093328233/Put/seqid=0 2024-04-14T11:15:29,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/799fdcf2fc07401880bae6e97adc0f5e 2024-04-14T11:15:29,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/baf8a0a169d44afb80c552eb1b0940dd is 50, key is test_row_0/C:col10/1713093328233/Put/seqid=0 2024-04-14T11:15:29,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/baf8a0a169d44afb80c552eb1b0940dd 2024-04-14T11:15:29,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093389618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:29,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:29,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093389619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093389619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:29,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093389619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:29,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093389619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:29,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/eaac03084b5240b788c680caeebf0929 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/eaac03084b5240b788c680caeebf0929 2024-04-14T11:15:29,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/eaac03084b5240b788c680caeebf0929, entries=150, sequenceid=114, filesize=11.7 K 2024-04-14T11:15:29,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/799fdcf2fc07401880bae6e97adc0f5e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/799fdcf2fc07401880bae6e97adc0f5e 2024-04-14T11:15:29,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/799fdcf2fc07401880bae6e97adc0f5e, entries=150, sequenceid=114, filesize=11.7 K 2024-04-14T11:15:29,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/baf8a0a169d44afb80c552eb1b0940dd as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/baf8a0a169d44afb80c552eb1b0940dd 2024-04-14T11:15:29,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/baf8a0a169d44afb80c552eb1b0940dd, entries=150, sequenceid=114, filesize=11.7 K 2024-04-14T11:15:29,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5407acadb5e9c9af78221a8f2d7e6785 in 218ms, sequenceid=114, compaction requested=true 2024-04-14T11:15:29,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:29,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:15:29,706 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:29,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:29,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:15:29,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:29,706 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:29,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:15:29,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:29,709 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36090 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:29,710 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/B is initiating minor compaction (all files) 2024-04-14T11:15:29,710 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/B in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:29,710 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/106ad7c3b715403681b187833e591a3b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c338873574c34a5dbf153b6ba866287d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/799fdcf2fc07401880bae6e97adc0f5e] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=35.2 K 2024-04-14T11:15:29,712 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38430 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:29,712 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/A is initiating minor compaction (all files) 2024-04-14T11:15:29,713 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/A in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:29,713 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/fca166abc2584f37b60e30b8a030cf0f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/2471ce883f8845beae4910af3241582f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/eaac03084b5240b788c680caeebf0929] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=37.5 K 2024-04-14T11:15:29,713 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 106ad7c3b715403681b187833e591a3b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1713093327770 2024-04-14T11:15:29,714 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting c338873574c34a5dbf153b6ba866287d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1713093327866 2024-04-14T11:15:29,715 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 799fdcf2fc07401880bae6e97adc0f5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1713093328233 2024-04-14T11:15:29,717 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting fca166abc2584f37b60e30b8a030cf0f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1713093327770 2024-04-14T11:15:29,717 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2471ce883f8845beae4910af3241582f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1713093327866 2024-04-14T11:15:29,718 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting eaac03084b5240b788c680caeebf0929, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1713093328233 2024-04-14T11:15:29,757 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#B#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:29,758 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/c15b907cd8284e0da65826d5a539747e is 50, key is test_row_0/B:col10/1713093328233/Put/seqid=0 2024-04-14T11:15:29,760 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#A#compaction#22 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:29,762 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/c594b3fcf2d242fe88ecd4277a2abf70 is 50, key is test_row_0/A:col10/1713093328233/Put/seqid=0 2024-04-14T11:15:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2024-04-14T11:15:29,826 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 12 completed 2024-04-14T11:15:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:29,828 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:29,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-04-14T11:15:29,831 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:15:29,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees 2024-04-14T11:15:29,836 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:15:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=14 2024-04-14T11:15:29,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/571194c6d49d4c87be9f2b65374acf59 is 50, key is test_row_0/A:col10/1713093329502/Put/seqid=0 2024-04-14T11:15:29,842 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=14, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:15:29,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:15:29,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093389896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:29,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093389899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:29,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093389900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:29,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:29,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093389901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:29,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:29,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093389902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=14 2024-04-14T11:15:30,000 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2024-04-14T11:15:30,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:30,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:30,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:30,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] handler.RSProcedureHandler(53): pid=15 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:30,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=15 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=15 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:30,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093390008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093390012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093390014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093390015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093390015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=14 2024-04-14T11:15:30,157 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2024-04-14T11:15:30,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:30,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:30,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:30,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] handler.RSProcedureHandler(53): pid=15 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:30,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=15 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=15 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:30,213 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/c15b907cd8284e0da65826d5a539747e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c15b907cd8284e0da65826d5a539747e 2024-04-14T11:15:30,216 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/c594b3fcf2d242fe88ecd4277a2abf70 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/c594b3fcf2d242fe88ecd4277a2abf70 2024-04-14T11:15:30,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093390217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093390218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093390218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093390219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093390220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,233 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/B of 5407acadb5e9c9af78221a8f2d7e6785 into c15b907cd8284e0da65826d5a539747e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:30,233 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/A of 5407acadb5e9c9af78221a8f2d7e6785 into c594b3fcf2d242fe88ecd4277a2abf70(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:30,233 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:30,233 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:30,233 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/B, priority=13, startTime=1713093329706; duration=0sec 2024-04-14T11:15:30,233 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/A, priority=13, startTime=1713093329706; duration=0sec 2024-04-14T11:15:30,234 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:30,234 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:30,234 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:A 2024-04-14T11:15:30,234 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:B 2024-04-14T11:15:30,234 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:30,238 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36090 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:30,238 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/C is initiating minor compaction (all files) 2024-04-14T11:15:30,238 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/C in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:30,238 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/f8952897c30b47ebb31cd6815503c07c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/953d1ddb0c644b88befffa1b900e6ecf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/baf8a0a169d44afb80c552eb1b0940dd] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=35.2 K 2024-04-14T11:15:30,239 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting f8952897c30b47ebb31cd6815503c07c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1713093327770 2024-04-14T11:15:30,240 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 953d1ddb0c644b88befffa1b900e6ecf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1713093327866 2024-04-14T11:15:30,243 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting baf8a0a169d44afb80c552eb1b0940dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1713093328233 2024-04-14T11:15:30,256 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#C#compaction#24 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:30,256 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/639031aa312b4b33a22d855a8eba60bd is 50, key is test_row_0/C:col10/1713093328233/Put/seqid=0 2024-04-14T11:15:30,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/571194c6d49d4c87be9f2b65374acf59 2024-04-14T11:15:30,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/f50ea39f038146d3a3db6b0d14b7c946 is 50, key is test_row_0/B:col10/1713093329502/Put/seqid=0 2024-04-14T11:15:30,313 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2024-04-14T11:15:30,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:30,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:30,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:30,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] handler.RSProcedureHandler(53): pid=15 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:30,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=15 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=15 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:30,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/f50ea39f038146d3a3db6b0d14b7c946 2024-04-14T11:15:30,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/d24b6633330543989f4de98465c80166 is 50, key is test_row_0/C:col10/1713093329502/Put/seqid=0 2024-04-14T11:15:30,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/d24b6633330543989f4de98465c80166 2024-04-14T11:15:30,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/571194c6d49d4c87be9f2b65374acf59 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/571194c6d49d4c87be9f2b65374acf59 2024-04-14T11:15:30,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/571194c6d49d4c87be9f2b65374acf59, entries=150, sequenceid=127, filesize=11.7 K 2024-04-14T11:15:30,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/f50ea39f038146d3a3db6b0d14b7c946 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/f50ea39f038146d3a3db6b0d14b7c946 2024-04-14T11:15:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=14 2024-04-14T11:15:30,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/f50ea39f038146d3a3db6b0d14b7c946, entries=150, sequenceid=127, filesize=11.7 K 2024-04-14T11:15:30,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/d24b6633330543989f4de98465c80166 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d24b6633330543989f4de98465c80166 2024-04-14T11:15:30,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d24b6633330543989f4de98465c80166, entries=150, sequenceid=127, filesize=11.7 K 2024-04-14T11:15:30,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 5407acadb5e9c9af78221a8f2d7e6785 in 642ms, sequenceid=127, compaction requested=false 2024-04-14T11:15:30,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:30,472 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2024-04-14T11:15:30,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:30,474 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-04-14T11:15:30,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/23410428c3204b3a912a55e693429a25 is 50, key is test_row_0/A:col10/1713093329897/Put/seqid=0 2024-04-14T11:15:30,517 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/23410428c3204b3a912a55e693429a25 2024-04-14T11:15:30,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:30,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/967246fad01f47af96f4b01288dc7c76 is 50, key is test_row_0/B:col10/1713093329897/Put/seqid=0 2024-04-14T11:15:30,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093390540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093390538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093390577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093390577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093390577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093390679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093390680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093390682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093390686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093390686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,697 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/639031aa312b4b33a22d855a8eba60bd as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/639031aa312b4b33a22d855a8eba60bd 2024-04-14T11:15:30,708 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-04-14T11:15:30,709 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-04-14T11:15:30,709 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/C of 5407acadb5e9c9af78221a8f2d7e6785 into 639031aa312b4b33a22d855a8eba60bd(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:30,710 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:30,710 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/C, priority=13, startTime=1713093329706; duration=0sec 2024-04-14T11:15:30,711 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:30,711 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:C 2024-04-14T11:15:30,712 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-04-14T11:15:30,712 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-04-14T11:15:30,713 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-04-14T11:15:30,713 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-04-14T11:15:30,713 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-04-14T11:15:30,713 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-04-14T11:15:30,715 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-04-14T11:15:30,715 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-04-14T11:15:30,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093390883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093390883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093390888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093390890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093390891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=14 2024-04-14T11:15:30,957 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/967246fad01f47af96f4b01288dc7c76 2024-04-14T11:15:30,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/4e230dad1d0b481189dc9fd5f9284630 is 50, key is test_row_0/C:col10/1713093329897/Put/seqid=0 2024-04-14T11:15:30,986 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/4e230dad1d0b481189dc9fd5f9284630 2024-04-14T11:15:31,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/23410428c3204b3a912a55e693429a25 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/23410428c3204b3a912a55e693429a25 2024-04-14T11:15:31,013 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/23410428c3204b3a912a55e693429a25, entries=150, sequenceid=152, filesize=11.8 K 2024-04-14T11:15:31,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/967246fad01f47af96f4b01288dc7c76 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/967246fad01f47af96f4b01288dc7c76 2024-04-14T11:15:31,032 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/967246fad01f47af96f4b01288dc7c76, entries=150, sequenceid=152, filesize=11.8 K 2024-04-14T11:15:31,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/4e230dad1d0b481189dc9fd5f9284630 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/4e230dad1d0b481189dc9fd5f9284630 2024-04-14T11:15:31,048 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/4e230dad1d0b481189dc9fd5f9284630, entries=150, sequenceid=152, filesize=11.8 K 2024-04-14T11:15:31,051 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(3035): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 5407acadb5e9c9af78221a8f2d7e6785 in 577ms, sequenceid=152, compaction requested=true 2024-04-14T11:15:31,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:31,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:31,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=15 2024-04-14T11:15:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=15 2024-04-14T11:15:31,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=15, resume processing ppid=14 2024-04-14T11:15:31,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=15, ppid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2120 sec 2024-04-14T11:15:31,064 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees in 1.2290 sec 2024-04-14T11:15:31,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:31,189 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:31,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-04-14T11:15:31,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/da3ffb172b8441ba81296a3bd44105cd is 50, key is test_row_0/A:col10/1713093331186/Put/seqid=0 2024-04-14T11:15:31,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/da3ffb172b8441ba81296a3bd44105cd 2024-04-14T11:15:31,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093391222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093391224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093391227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093391227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/6beec88b1aa34e1f84db1955b37fe958 is 50, key is test_row_0/B:col10/1713093331186/Put/seqid=0 2024-04-14T11:15:31,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093391228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/6beec88b1aa34e1f84db1955b37fe958 2024-04-14T11:15:31,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/882e5627eb904d1eade4afbd61bfdfec is 50, key is test_row_0/C:col10/1713093331186/Put/seqid=0 2024-04-14T11:15:31,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/882e5627eb904d1eade4afbd61bfdfec 2024-04-14T11:15:31,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/da3ffb172b8441ba81296a3bd44105cd as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/da3ffb172b8441ba81296a3bd44105cd 2024-04-14T11:15:31,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/da3ffb172b8441ba81296a3bd44105cd, entries=250, sequenceid=169, filesize=16.5 K 2024-04-14T11:15:31,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/6beec88b1aa34e1f84db1955b37fe958 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/6beec88b1aa34e1f84db1955b37fe958 2024-04-14T11:15:31,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/6beec88b1aa34e1f84db1955b37fe958, entries=150, sequenceid=169, filesize=11.8 K 2024-04-14T11:15:31,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/882e5627eb904d1eade4afbd61bfdfec as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/882e5627eb904d1eade4afbd61bfdfec 2024-04-14T11:15:31,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/882e5627eb904d1eade4afbd61bfdfec, entries=150, sequenceid=169, filesize=11.8 K 2024-04-14T11:15:31,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=127.47 KB/130530 for 5407acadb5e9c9af78221a8f2d7e6785 in 146ms, sequenceid=169, compaction requested=true 2024-04-14T11:15:31,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:31,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:15:31,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:31,336 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:15:31,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:15:31,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:31,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:15:31,336 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:15:31,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:31,339 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53256 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:15:31,339 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/A is initiating minor compaction (all files) 2024-04-14T11:15:31,339 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/A in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:31,339 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/c594b3fcf2d242fe88ecd4277a2abf70, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/571194c6d49d4c87be9f2b65374acf59, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/23410428c3204b3a912a55e693429a25, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/da3ffb172b8441ba81296a3bd44105cd] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=52.0 K 2024-04-14T11:15:31,340 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:15:31,340 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/B is initiating minor compaction (all files) 2024-04-14T11:15:31,340 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/B in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:31,340 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c15b907cd8284e0da65826d5a539747e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/f50ea39f038146d3a3db6b0d14b7c946, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/967246fad01f47af96f4b01288dc7c76, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/6beec88b1aa34e1f84db1955b37fe958] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=47.3 K 2024-04-14T11:15:31,340 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c594b3fcf2d242fe88ecd4277a2abf70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1713093328233 2024-04-14T11:15:31,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:31,341 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:31,341 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting c15b907cd8284e0da65826d5a539747e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1713093328233 2024-04-14T11:15:31,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-04-14T11:15:31,342 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 571194c6d49d4c87be9f2b65374acf59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1713093329502 2024-04-14T11:15:31,342 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting f50ea39f038146d3a3db6b0d14b7c946, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1713093329502 2024-04-14T11:15:31,343 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 23410428c3204b3a912a55e693429a25, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1713093329895 2024-04-14T11:15:31,343 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 967246fad01f47af96f4b01288dc7c76, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1713093329895 2024-04-14T11:15:31,343 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting da3ffb172b8441ba81296a3bd44105cd, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1713093330529 2024-04-14T11:15:31,344 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 6beec88b1aa34e1f84db1955b37fe958, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1713093330550 2024-04-14T11:15:31,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/7faf64ea0c854a7aa777bd63bcc61279 is 50, key is test_row_0/A:col10/1713093331223/Put/seqid=0 2024-04-14T11:15:31,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4522) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4453) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:4942) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:4936) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:4932) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3228) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093391353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093391362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093391362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093391362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,373 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#A#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:31,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,374 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/815ca5f592ef4872b34b0685988d2e52 is 50, key is test_row_0/A:col10/1713093331186/Put/seqid=0 2024-04-14T11:15:31,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093391366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,379 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#B#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:31,380 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/82ce418fe4ec495d998036f8a6852395 is 50, key is test_row_0/B:col10/1713093331186/Put/seqid=0 2024-04-14T11:15:31,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/7faf64ea0c854a7aa777bd63bcc61279 2024-04-14T11:15:31,428 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/815ca5f592ef4872b34b0685988d2e52 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/815ca5f592ef4872b34b0685988d2e52 2024-04-14T11:15:31,441 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/A of 5407acadb5e9c9af78221a8f2d7e6785 into 815ca5f592ef4872b34b0685988d2e52(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:31,441 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:31,441 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/A, priority=12, startTime=1713093331335; duration=0sec 2024-04-14T11:15:31,441 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:31,441 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:A 2024-04-14T11:15:31,441 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:15:31,447 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:15:31,447 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/C is initiating minor compaction (all files) 2024-04-14T11:15:31,447 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/C in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:31,447 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/639031aa312b4b33a22d855a8eba60bd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d24b6633330543989f4de98465c80166, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/4e230dad1d0b481189dc9fd5f9284630, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/882e5627eb904d1eade4afbd61bfdfec] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=47.3 K 2024-04-14T11:15:31,449 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 639031aa312b4b33a22d855a8eba60bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1713093328233 2024-04-14T11:15:31,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/c79e28636b8848ceb7bd8bdcb7b17eab is 50, key is test_row_0/B:col10/1713093331223/Put/seqid=0 2024-04-14T11:15:31,449 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting d24b6633330543989f4de98465c80166, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1713093329502 2024-04-14T11:15:31,453 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4e230dad1d0b481189dc9fd5f9284630, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1713093329895 2024-04-14T11:15:31,454 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 882e5627eb904d1eade4afbd61bfdfec, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1713093330550 2024-04-14T11:15:31,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/c79e28636b8848ceb7bd8bdcb7b17eab 2024-04-14T11:15:31,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093391469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093391470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093391482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093391483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,487 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#C#compaction#37 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:31,488 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/2237c363f8c1425c8d301bdb40368661 is 50, key is test_row_0/C:col10/1713093331186/Put/seqid=0 2024-04-14T11:15:31,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093391490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/bfea52c14f784dedbb596470764f4516 is 50, key is test_row_0/C:col10/1713093331223/Put/seqid=0 2024-04-14T11:15:31,504 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/2237c363f8c1425c8d301bdb40368661 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/2237c363f8c1425c8d301bdb40368661 2024-04-14T11:15:31,514 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/C of 5407acadb5e9c9af78221a8f2d7e6785 into 2237c363f8c1425c8d301bdb40368661(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:31,514 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:31,514 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/C, priority=12, startTime=1713093331336; duration=0sec 2024-04-14T11:15:31,515 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:31,515 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:C 2024-04-14T11:15:31,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093391685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093391686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093391686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093391688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093391692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:31,824 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/82ce418fe4ec495d998036f8a6852395 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/82ce418fe4ec495d998036f8a6852395 2024-04-14T11:15:31,834 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/B of 5407acadb5e9c9af78221a8f2d7e6785 into 82ce418fe4ec495d998036f8a6852395(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:31,835 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:31,835 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/B, priority=12, startTime=1713093331336; duration=0sec 2024-04-14T11:15:31,835 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:31,835 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:B 2024-04-14T11:15:31,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/bfea52c14f784dedbb596470764f4516 2024-04-14T11:15:31,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/7faf64ea0c854a7aa777bd63bcc61279 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/7faf64ea0c854a7aa777bd63bcc61279 2024-04-14T11:15:31,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/7faf64ea0c854a7aa777bd63bcc61279, entries=150, sequenceid=193, filesize=11.8 K 2024-04-14T11:15:31,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/c79e28636b8848ceb7bd8bdcb7b17eab as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c79e28636b8848ceb7bd8bdcb7b17eab 2024-04-14T11:15:31,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=14 2024-04-14T11:15:31,948 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 14 completed 2024-04-14T11:15:31,950 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:15:31,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees 2024-04-14T11:15:31,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2024-04-14T11:15:31,955 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:15:31,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c79e28636b8848ceb7bd8bdcb7b17eab, entries=150, sequenceid=193, filesize=11.8 K 2024-04-14T11:15:31,956 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=16, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:15:31,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:15:31,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/bfea52c14f784dedbb596470764f4516 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/bfea52c14f784dedbb596470764f4516 2024-04-14T11:15:31,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/bfea52c14f784dedbb596470764f4516, entries=150, sequenceid=193, filesize=11.8 K 2024-04-14T11:15:31,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5407acadb5e9c9af78221a8f2d7e6785 in 628ms, sequenceid=193, compaction requested=false 2024-04-14T11:15:31,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:31,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:31,992 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:31,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-04-14T11:15:32,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/50d4d8d7b8614e4ea7babc6ca6bf0069 is 50, key is test_row_0/A:col10/1713093331991/Put/seqid=0 2024-04-14T11:15:32,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/50d4d8d7b8614e4ea7babc6ca6bf0069 2024-04-14T11:15:32,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/3b9645903ef348c3a0cba98ae5233113 is 50, key is test_row_0/B:col10/1713093331991/Put/seqid=0 2024-04-14T11:15:32,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093392030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093392022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093392032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093392032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093392032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/3b9645903ef348c3a0cba98ae5233113 2024-04-14T11:15:32,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/5159079a347740da8ff362dc143c5643 is 50, key is test_row_0/C:col10/1713093331991/Put/seqid=0 2024-04-14T11:15:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2024-04-14T11:15:32,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/5159079a347740da8ff362dc143c5643 2024-04-14T11:15:32,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/50d4d8d7b8614e4ea7babc6ca6bf0069 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/50d4d8d7b8614e4ea7babc6ca6bf0069 2024-04-14T11:15:32,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/50d4d8d7b8614e4ea7babc6ca6bf0069, entries=150, sequenceid=211, filesize=11.8 K 2024-04-14T11:15:32,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/3b9645903ef348c3a0cba98ae5233113 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/3b9645903ef348c3a0cba98ae5233113 2024-04-14T11:15:32,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/3b9645903ef348c3a0cba98ae5233113, entries=150, sequenceid=211, filesize=11.8 K 2024-04-14T11:15:32,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/5159079a347740da8ff362dc143c5643 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/5159079a347740da8ff362dc143c5643 2024-04-14T11:15:32,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/5159079a347740da8ff362dc143c5643, entries=150, sequenceid=211, filesize=11.8 K 2024-04-14T11:15:32,109 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 5407acadb5e9c9af78221a8f2d7e6785 in 118ms, sequenceid=211, compaction requested=true 2024-04-14T11:15:32,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:32,111 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:32,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:15:32,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:32,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:15:32,111 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:32,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2024-04-14T11:15:32,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:32,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:32,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:15:32,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:32,112 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-04-14T11:15:32,114 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36778 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:32,114 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/B is initiating minor compaction (all files) 2024-04-14T11:15:32,114 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/B in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:32,114 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/82ce418fe4ec495d998036f8a6852395, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c79e28636b8848ceb7bd8bdcb7b17eab, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/3b9645903ef348c3a0cba98ae5233113] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=35.9 K 2024-04-14T11:15:32,114 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36778 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:32,114 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/A is initiating minor compaction (all files) 2024-04-14T11:15:32,115 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/A in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:32,115 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/815ca5f592ef4872b34b0685988d2e52, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/7faf64ea0c854a7aa777bd63bcc61279, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/50d4d8d7b8614e4ea7babc6ca6bf0069] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=35.9 K 2024-04-14T11:15:32,116 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 815ca5f592ef4872b34b0685988d2e52, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1713093330550 2024-04-14T11:15:32,117 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 82ce418fe4ec495d998036f8a6852395, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1713093330550 2024-04-14T11:15:32,117 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 7faf64ea0c854a7aa777bd63bcc61279, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1713093331223 2024-04-14T11:15:32,118 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 50d4d8d7b8614e4ea7babc6ca6bf0069, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1713093331989 2024-04-14T11:15:32,118 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c79e28636b8848ceb7bd8bdcb7b17eab, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1713093331223 2024-04-14T11:15:32,119 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3b9645903ef348c3a0cba98ae5233113, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1713093331989 2024-04-14T11:15:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:32,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:32,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/cecaf62a39974a2ea37fbaaf4ea9e884 is 50, key is test_row_0/A:col10/1713093332028/Put/seqid=0 2024-04-14T11:15:32,145 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#B#compaction#44 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:32,145 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#A#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:32,146 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/6181033228884261a35a4b48e6142e85 is 50, key is test_row_0/B:col10/1713093331991/Put/seqid=0 2024-04-14T11:15:32,146 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/71660db0547d452e9043f211ede5c674 is 50, key is test_row_0/A:col10/1713093331991/Put/seqid=0 2024-04-14T11:15:32,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093392159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093392161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093392162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093392165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093392166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,175 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/cecaf62a39974a2ea37fbaaf4ea9e884 2024-04-14T11:15:32,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/9592fbea2d6c48fa9fcc27b8a9bdfeb5 is 50, key is test_row_0/B:col10/1713093332028/Put/seqid=0 2024-04-14T11:15:32,220 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/71660db0547d452e9043f211ede5c674 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/71660db0547d452e9043f211ede5c674 2024-04-14T11:15:32,222 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/6181033228884261a35a4b48e6142e85 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/6181033228884261a35a4b48e6142e85 2024-04-14T11:15:32,227 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/9592fbea2d6c48fa9fcc27b8a9bdfeb5 2024-04-14T11:15:32,239 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/A of 5407acadb5e9c9af78221a8f2d7e6785 into 71660db0547d452e9043f211ede5c674(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:32,239 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:32,240 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/A, priority=13, startTime=1713093332110; duration=0sec 2024-04-14T11:15:32,240 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:32,240 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:A 2024-04-14T11:15:32,240 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:32,243 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36778 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:32,243 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/C is initiating minor compaction (all files) 2024-04-14T11:15:32,243 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/C in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:32,243 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/B of 5407acadb5e9c9af78221a8f2d7e6785 into 6181033228884261a35a4b48e6142e85(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:32,243 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/2237c363f8c1425c8d301bdb40368661, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/bfea52c14f784dedbb596470764f4516, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/5159079a347740da8ff362dc143c5643] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=35.9 K 2024-04-14T11:15:32,244 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:32,244 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/B, priority=13, startTime=1713093332111; duration=0sec 2024-04-14T11:15:32,244 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:32,244 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:B 2024-04-14T11:15:32,245 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 2237c363f8c1425c8d301bdb40368661, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1713093330550 2024-04-14T11:15:32,246 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting bfea52c14f784dedbb596470764f4516, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1713093331223 2024-04-14T11:15:32,247 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 5159079a347740da8ff362dc143c5643, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1713093331989 2024-04-14T11:15:32,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/41a8b3ea7f7f43aeb057d7340643a06b is 50, key is test_row_0/C:col10/1713093332028/Put/seqid=0 2024-04-14T11:15:32,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2024-04-14T11:15:32,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093392267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093392269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093392270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093392273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093392274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,292 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#C#compaction#47 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:32,293 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/9f6c79a221a144a0a7e5ef01f1af215f is 50, key is test_row_0/C:col10/1713093331991/Put/seqid=0 2024-04-14T11:15:32,303 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/41a8b3ea7f7f43aeb057d7340643a06b 2024-04-14T11:15:32,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/cecaf62a39974a2ea37fbaaf4ea9e884 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cecaf62a39974a2ea37fbaaf4ea9e884 2024-04-14T11:15:32,323 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cecaf62a39974a2ea37fbaaf4ea9e884, entries=150, sequenceid=232, filesize=11.8 K 2024-04-14T11:15:32,325 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/9f6c79a221a144a0a7e5ef01f1af215f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/9f6c79a221a144a0a7e5ef01f1af215f 2024-04-14T11:15:32,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/9592fbea2d6c48fa9fcc27b8a9bdfeb5 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/9592fbea2d6c48fa9fcc27b8a9bdfeb5 2024-04-14T11:15:32,334 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/C of 5407acadb5e9c9af78221a8f2d7e6785 into 9f6c79a221a144a0a7e5ef01f1af215f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:32,334 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:32,335 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/C, priority=13, startTime=1713093332111; duration=0sec 2024-04-14T11:15:32,335 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:32,335 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/9592fbea2d6c48fa9fcc27b8a9bdfeb5, entries=150, sequenceid=232, filesize=11.8 K 2024-04-14T11:15:32,335 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:C 2024-04-14T11:15:32,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/41a8b3ea7f7f43aeb057d7340643a06b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/41a8b3ea7f7f43aeb057d7340643a06b 2024-04-14T11:15:32,353 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/41a8b3ea7f7f43aeb057d7340643a06b, entries=150, sequenceid=232, filesize=11.8 K 2024-04-14T11:15:32,356 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(3035): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 5407acadb5e9c9af78221a8f2d7e6785 in 244ms, sequenceid=232, compaction requested=false 2024-04-14T11:15:32,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:32,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:32,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=17 2024-04-14T11:15:32,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=17 2024-04-14T11:15:32,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=17, resume processing ppid=16 2024-04-14T11:15:32,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=17, ppid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 403 msec 2024-04-14T11:15:32,367 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees in 415 msec 2024-04-14T11:15:32,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:32,474 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:32,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-04-14T11:15:32,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/55784b1558dd46f18ec089dba5bf229d is 50, key is test_row_0/A:col10/1713093332163/Put/seqid=0 2024-04-14T11:15:32,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093392494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/55784b1558dd46f18ec089dba5bf229d 2024-04-14T11:15:32,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093392498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093392499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093392501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093392510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/078dfeee46304bd981d3bfa15b77e68a is 50, key is test_row_0/B:col10/1713093332163/Put/seqid=0 2024-04-14T11:15:32,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2024-04-14T11:15:32,560 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 16 completed 2024-04-14T11:15:32,563 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:15:32,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees 2024-04-14T11:15:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=18 2024-04-14T11:15:32,568 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:15:32,569 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=18, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:15:32,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:15:32,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093392601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093392609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093392609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093392613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093392618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=18 2024-04-14T11:15:32,722 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-04-14T11:15:32,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:32,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:32,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:32,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(53): pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:32,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:32,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093392808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093392814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093392814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093392816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:32,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093392820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=18 2024-04-14T11:15:32,880 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:32,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-04-14T11:15:32,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:32,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:32,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:32,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(53): pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:32,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:32,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:32,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/078dfeee46304bd981d3bfa15b77e68a 2024-04-14T11:15:32,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/c57ee0796325455f9cbf0f789243ef3e is 50, key is test_row_0/C:col10/1713093332163/Put/seqid=0 2024-04-14T11:15:33,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/c57ee0796325455f9cbf0f789243ef3e 2024-04-14T11:15:33,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/55784b1558dd46f18ec089dba5bf229d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/55784b1558dd46f18ec089dba5bf229d 2024-04-14T11:15:33,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/55784b1558dd46f18ec089dba5bf229d, entries=200, sequenceid=252, filesize=14.2 K 2024-04-14T11:15:33,035 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-04-14T11:15:33,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:33,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:33,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:33,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(53): pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/078dfeee46304bd981d3bfa15b77e68a as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/078dfeee46304bd981d3bfa15b77e68a 2024-04-14T11:15:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/078dfeee46304bd981d3bfa15b77e68a, entries=150, sequenceid=252, filesize=11.8 K 2024-04-14T11:15:33,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/c57ee0796325455f9cbf0f789243ef3e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/c57ee0796325455f9cbf0f789243ef3e 2024-04-14T11:15:33,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/c57ee0796325455f9cbf0f789243ef3e, entries=150, sequenceid=252, filesize=11.8 K 2024-04-14T11:15:33,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 5407acadb5e9c9af78221a8f2d7e6785 in 592ms, sequenceid=252, compaction requested=true 2024-04-14T11:15:33,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:33,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:15:33,067 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:33,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:33,067 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:33,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:15:33,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:33,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:15:33,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:33,070 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39270 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:33,070 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/A is initiating minor compaction (all files) 2024-04-14T11:15:33,070 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/A in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:33,070 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36880 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:33,070 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/71660db0547d452e9043f211ede5c674, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cecaf62a39974a2ea37fbaaf4ea9e884, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/55784b1558dd46f18ec089dba5bf229d] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=38.3 K 2024-04-14T11:15:33,070 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/B is initiating minor compaction (all files) 2024-04-14T11:15:33,070 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/B in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:33,070 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/6181033228884261a35a4b48e6142e85, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/9592fbea2d6c48fa9fcc27b8a9bdfeb5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/078dfeee46304bd981d3bfa15b77e68a] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=36.0 K 2024-04-14T11:15:33,071 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 71660db0547d452e9043f211ede5c674, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1713093331989 2024-04-14T11:15:33,073 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 6181033228884261a35a4b48e6142e85, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1713093331989 2024-04-14T11:15:33,073 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting cecaf62a39974a2ea37fbaaf4ea9e884, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1713093332022 2024-04-14T11:15:33,074 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 55784b1558dd46f18ec089dba5bf229d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1713093332154 2024-04-14T11:15:33,074 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 9592fbea2d6c48fa9fcc27b8a9bdfeb5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1713093332022 2024-04-14T11:15:33,075 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 078dfeee46304bd981d3bfa15b77e68a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1713093332162 2024-04-14T11:15:33,100 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#A#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:33,101 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/cf92cb3750ac45e5862b50a733ffc624 is 50, key is test_row_0/A:col10/1713093332163/Put/seqid=0 2024-04-14T11:15:33,111 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#B#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:33,112 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/ef5271ae8d9c4c2cb2923aab89027bdf is 50, key is test_row_0/B:col10/1713093332163/Put/seqid=0 2024-04-14T11:15:33,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:33,117 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:33,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-04-14T11:15:33,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/5e95cac412e040e0b7eb8478f849b603 is 50, key is test_row_0/A:col10/1713093332496/Put/seqid=0 2024-04-14T11:15:33,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093393139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093393142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093393142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093393143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093393143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/5e95cac412e040e0b7eb8478f849b603 2024-04-14T11:15:33,158 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/cf92cb3750ac45e5862b50a733ffc624 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cf92cb3750ac45e5862b50a733ffc624 2024-04-14T11:15:33,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=18 2024-04-14T11:15:33,173 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/ef5271ae8d9c4c2cb2923aab89027bdf as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/ef5271ae8d9c4c2cb2923aab89027bdf 2024-04-14T11:15:33,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/be17b3be30a34446abc1547814716abc is 50, key is test_row_0/B:col10/1713093332496/Put/seqid=0 2024-04-14T11:15:33,181 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/A of 5407acadb5e9c9af78221a8f2d7e6785 into cf92cb3750ac45e5862b50a733ffc624(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:33,181 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:33,181 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/A, priority=13, startTime=1713093333067; duration=0sec 2024-04-14T11:15:33,181 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:33,181 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:A 2024-04-14T11:15:33,181 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:33,189 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/B of 5407acadb5e9c9af78221a8f2d7e6785 into ef5271ae8d9c4c2cb2923aab89027bdf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:33,189 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:33,189 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/B, priority=13, startTime=1713093333067; duration=0sec 2024-04-14T11:15:33,189 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:33,189 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:B 2024-04-14T11:15:33,192 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36880 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:33,193 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/C is initiating minor compaction (all files) 2024-04-14T11:15:33,193 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,193 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/C in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:33,193 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/9f6c79a221a144a0a7e5ef01f1af215f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/41a8b3ea7f7f43aeb057d7340643a06b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/c57ee0796325455f9cbf0f789243ef3e] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=36.0 K 2024-04-14T11:15:33,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-04-14T11:15:33,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:33,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:33,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:33,194 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9f6c79a221a144a0a7e5ef01f1af215f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1713093331989 2024-04-14T11:15:33,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(53): pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,195 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 41a8b3ea7f7f43aeb057d7340643a06b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1713093332022 2024-04-14T11:15:33,196 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c57ee0796325455f9cbf0f789243ef3e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1713093332162 2024-04-14T11:15:33,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/be17b3be30a34446abc1547814716abc 2024-04-14T11:15:33,221 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#C#compaction#55 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:33,222 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/1e08ac2e49d24d2a880faf3191a1734c is 50, key is test_row_0/C:col10/1713093332163/Put/seqid=0 2024-04-14T11:15:33,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/d9864ac9d5db4600972d8564e4d0b7cc is 50, key is test_row_0/C:col10/1713093332496/Put/seqid=0 2024-04-14T11:15:33,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093393246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093393253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093393254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093393254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093393255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,356 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-04-14T11:15:33,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:33,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:33,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:33,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(53): pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093393451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093393457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093393457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093393460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093393461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,511 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-04-14T11:15:33,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:33,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:33,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:33,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(53): pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,641 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/1e08ac2e49d24d2a880faf3191a1734c as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/1e08ac2e49d24d2a880faf3191a1734c 2024-04-14T11:15:33,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/d9864ac9d5db4600972d8564e4d0b7cc 2024-04-14T11:15:33,662 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/C of 5407acadb5e9c9af78221a8f2d7e6785 into 1e08ac2e49d24d2a880faf3191a1734c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:33,662 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:33,662 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/C, priority=13, startTime=1713093333067; duration=0sec 2024-04-14T11:15:33,663 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:33,663 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:C 2024-04-14T11:15:33,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/5e95cac412e040e0b7eb8478f849b603 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/5e95cac412e040e0b7eb8478f849b603 2024-04-14T11:15:33,666 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-04-14T11:15:33,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:33,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:33,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:33,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(53): pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=18 2024-04-14T11:15:33,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/5e95cac412e040e0b7eb8478f849b603, entries=200, sequenceid=273, filesize=14.4 K 2024-04-14T11:15:33,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/be17b3be30a34446abc1547814716abc as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/be17b3be30a34446abc1547814716abc 2024-04-14T11:15:33,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/be17b3be30a34446abc1547814716abc, entries=150, sequenceid=273, filesize=12.0 K 2024-04-14T11:15:33,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/d9864ac9d5db4600972d8564e4d0b7cc as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d9864ac9d5db4600972d8564e4d0b7cc 2024-04-14T11:15:33,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d9864ac9d5db4600972d8564e4d0b7cc, entries=150, sequenceid=273, filesize=12.0 K 2024-04-14T11:15:33,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 5407acadb5e9c9af78221a8f2d7e6785 in 582ms, sequenceid=273, compaction requested=false 2024-04-14T11:15:33,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:33,757 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:33,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-04-14T11:15:33,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/e1a80d1d0d3a42d1bda76494dcf07baf is 50, key is test_row_0/A:col10/1713093333141/Put/seqid=0 2024-04-14T11:15:33,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/e1a80d1d0d3a42d1bda76494dcf07baf 2024-04-14T11:15:33,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093393794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093393797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093393797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093393800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093393800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/06abaa706f8d49fd96c49455319c5adf is 50, key is test_row_0/B:col10/1713093333141/Put/seqid=0 2024-04-14T11:15:33,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/06abaa706f8d49fd96c49455319c5adf 2024-04-14T11:15:33,824 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-04-14T11:15:33,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:33,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:33,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:33,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(53): pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/5dfa14b0c9994e87a34e9d374d017087 is 50, key is test_row_0/C:col10/1713093333141/Put/seqid=0 2024-04-14T11:15:33,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093393902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093393902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093393904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093393905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093393905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,980 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:33,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-04-14T11:15:33,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:33,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:33,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:33,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(53): pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:33,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:34,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093394108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093394108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093394109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093394111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093394111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,136 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-04-14T11:15:34,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:34,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:34,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:34,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(53): pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:34,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:34,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:34,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/5dfa14b0c9994e87a34e9d374d017087 2024-04-14T11:15:34,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/e1a80d1d0d3a42d1bda76494dcf07baf as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e1a80d1d0d3a42d1bda76494dcf07baf 2024-04-14T11:15:34,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e1a80d1d0d3a42d1bda76494dcf07baf, entries=150, sequenceid=293, filesize=12.0 K 2024-04-14T11:15:34,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/06abaa706f8d49fd96c49455319c5adf as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/06abaa706f8d49fd96c49455319c5adf 2024-04-14T11:15:34,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/06abaa706f8d49fd96c49455319c5adf, entries=150, sequenceid=293, filesize=12.0 K 2024-04-14T11:15:34,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/5dfa14b0c9994e87a34e9d374d017087 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/5dfa14b0c9994e87a34e9d374d017087 2024-04-14T11:15:34,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/5dfa14b0c9994e87a34e9d374d017087, entries=150, sequenceid=293, filesize=12.0 K 2024-04-14T11:15:34,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 5407acadb5e9c9af78221a8f2d7e6785 in 528ms, sequenceid=293, compaction requested=true 2024-04-14T11:15:34,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:34,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:15:34,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:34,286 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:34,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:15:34,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:34,286 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:34,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:15:34,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:34,287 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39722 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:34,288 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/A is initiating minor compaction (all files) 2024-04-14T11:15:34,288 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/A in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:34,288 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cf92cb3750ac45e5862b50a733ffc624, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/5e95cac412e040e0b7eb8478f849b603, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e1a80d1d0d3a42d1bda76494dcf07baf] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=38.8 K 2024-04-14T11:15:34,288 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37282 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:34,288 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/B is initiating minor compaction (all files) 2024-04-14T11:15:34,288 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting cf92cb3750ac45e5862b50a733ffc624, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1713093332162 2024-04-14T11:15:34,288 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/B in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:34,288 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/ef5271ae8d9c4c2cb2923aab89027bdf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/be17b3be30a34446abc1547814716abc, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/06abaa706f8d49fd96c49455319c5adf] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=36.4 K 2024-04-14T11:15:34,289 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5e95cac412e040e0b7eb8478f849b603, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1713093332492 2024-04-14T11:15:34,289 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting ef5271ae8d9c4c2cb2923aab89027bdf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1713093332162 2024-04-14T11:15:34,290 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting e1a80d1d0d3a42d1bda76494dcf07baf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1713093333140 2024-04-14T11:15:34,290 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting be17b3be30a34446abc1547814716abc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1713093332496 2024-04-14T11:15:34,290 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,291 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 06abaa706f8d49fd96c49455319c5adf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1713093333140 2024-04-14T11:15:34,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-04-14T11:15:34,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:34,291 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-04-14T11:15:34,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/d8f6bc1a89684f939e31c7e2de9ccdb3 is 50, key is test_row_0/A:col10/1713093333794/Put/seqid=0 2024-04-14T11:15:34,315 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#B#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:34,316 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/58e7ab9d98734338bdd1979179878189 is 50, key is test_row_0/B:col10/1713093333141/Put/seqid=0 2024-04-14T11:15:34,319 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#A#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:34,320 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/e372b9d0582b4a6c97dc34d7758b6243 is 50, key is test_row_0/A:col10/1713093333141/Put/seqid=0 2024-04-14T11:15:34,339 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/d8f6bc1a89684f939e31c7e2de9ccdb3 2024-04-14T11:15:34,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/c92d4011bfd54413be80fe3e2303b473 is 50, key is test_row_0/B:col10/1713093333794/Put/seqid=0 2024-04-14T11:15:34,371 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/c92d4011bfd54413be80fe3e2303b473 2024-04-14T11:15:34,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/8d94324f81244e4291ae886defcc0fc0 is 50, key is test_row_0/C:col10/1713093333794/Put/seqid=0 2024-04-14T11:15:34,395 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/8d94324f81244e4291ae886defcc0fc0 2024-04-14T11:15:34,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/d8f6bc1a89684f939e31c7e2de9ccdb3 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d8f6bc1a89684f939e31c7e2de9ccdb3 2024-04-14T11:15:34,416 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d8f6bc1a89684f939e31c7e2de9ccdb3, entries=150, sequenceid=312, filesize=12.0 K 2024-04-14T11:15:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:34,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:34,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/c92d4011bfd54413be80fe3e2303b473 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c92d4011bfd54413be80fe3e2303b473 2024-04-14T11:15:34,433 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c92d4011bfd54413be80fe3e2303b473, entries=150, sequenceid=312, filesize=12.0 K 2024-04-14T11:15:34,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/8d94324f81244e4291ae886defcc0fc0 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/8d94324f81244e4291ae886defcc0fc0 2024-04-14T11:15:34,444 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/8d94324f81244e4291ae886defcc0fc0, entries=150, sequenceid=312, filesize=12.0 K 2024-04-14T11:15:34,446 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(3035): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=87.22 KB/89310 for 5407acadb5e9c9af78221a8f2d7e6785 in 155ms, sequenceid=312, compaction requested=true 2024-04-14T11:15:34,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:34,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:34,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=19 2024-04-14T11:15:34,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=19 2024-04-14T11:15:34,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:34,448 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:34,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-04-14T11:15:34,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=19, resume processing ppid=18 2024-04-14T11:15:34,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=19, ppid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8790 sec 2024-04-14T11:15:34,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees in 1.8920 sec 2024-04-14T11:15:34,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/b882ad39904f48b3bb8ae6f6dcd145d8 is 50, key is test_row_0/A:col10/1713093334448/Put/seqid=0 2024-04-14T11:15:34,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/b882ad39904f48b3bb8ae6f6dcd145d8 2024-04-14T11:15:34,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093394477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093394479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093394480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093394482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093394482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/af22681d8e844e83a572d5dfce7e79f4 is 50, key is test_row_0/B:col10/1713093334448/Put/seqid=0 2024-04-14T11:15:34,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/af22681d8e844e83a572d5dfce7e79f4 2024-04-14T11:15:34,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/909ff096419d48bdb07670703fb81c07 is 50, key is test_row_0/C:col10/1713093334448/Put/seqid=0 2024-04-14T11:15:34,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/909ff096419d48bdb07670703fb81c07 2024-04-14T11:15:34,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/b882ad39904f48b3bb8ae6f6dcd145d8 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/b882ad39904f48b3bb8ae6f6dcd145d8 2024-04-14T11:15:34,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/b882ad39904f48b3bb8ae6f6dcd145d8, entries=200, sequenceid=330, filesize=14.4 K 2024-04-14T11:15:34,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/af22681d8e844e83a572d5dfce7e79f4 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/af22681d8e844e83a572d5dfce7e79f4 2024-04-14T11:15:34,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093394584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/af22681d8e844e83a572d5dfce7e79f4, entries=150, sequenceid=330, filesize=12.0 K 2024-04-14T11:15:34,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093394588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/909ff096419d48bdb07670703fb81c07 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/909ff096419d48bdb07670703fb81c07 2024-04-14T11:15:34,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093394592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093394593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093394593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/909ff096419d48bdb07670703fb81c07, entries=150, sequenceid=330, filesize=12.0 K 2024-04-14T11:15:34,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 5407acadb5e9c9af78221a8f2d7e6785 in 157ms, sequenceid=330, compaction requested=true 2024-04-14T11:15:34,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:34,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:A, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:15:34,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-04-14T11:15:34,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:B, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:15:34,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-04-14T11:15:34,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:15:34,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-04-14T11:15:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=18 2024-04-14T11:15:34,676 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 18 completed 2024-04-14T11:15:34,678 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:15:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees 2024-04-14T11:15:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=20 2024-04-14T11:15:34,682 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:15:34,683 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=20, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:15:34,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:15:34,760 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/58e7ab9d98734338bdd1979179878189 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/58e7ab9d98734338bdd1979179878189 2024-04-14T11:15:34,764 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/e372b9d0582b4a6c97dc34d7758b6243 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e372b9d0582b4a6c97dc34d7758b6243 2024-04-14T11:15:34,775 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/B of 5407acadb5e9c9af78221a8f2d7e6785 into 58e7ab9d98734338bdd1979179878189(size=12.7 K), total size for store is 36.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:34,775 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:34,775 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/B, priority=13, startTime=1713093334286; duration=0sec 2024-04-14T11:15:34,776 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-04-14T11:15:34,776 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:B 2024-04-14T11:15:34,776 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-04-14T11:15:34,778 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-04-14T11:15:34,778 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-04-14T11:15:34,778 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. because compaction request was cancelled 2024-04-14T11:15:34,778 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:A 2024-04-14T11:15:34,778 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-04-14T11:15:34,778 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/A of 5407acadb5e9c9af78221a8f2d7e6785 into e372b9d0582b4a6c97dc34d7758b6243(size=12.7 K), total size for store is 39.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:34,778 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:34,778 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/A, priority=13, startTime=1713093334285; duration=0sec 2024-04-14T11:15:34,778 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-04-14T11:15:34,778 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:A 2024-04-14T11:15:34,779 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:B 2024-04-14T11:15:34,781 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61850 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-04-14T11:15:34,781 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/C is initiating minor compaction (all files) 2024-04-14T11:15:34,781 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-04-14T11:15:34,781 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-04-14T11:15:34,781 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/C in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:34,781 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-04-14T11:15:34,781 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. because compaction request was cancelled 2024-04-14T11:15:34,781 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:C 2024-04-14T11:15:34,781 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/1e08ac2e49d24d2a880faf3191a1734c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d9864ac9d5db4600972d8564e4d0b7cc, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/5dfa14b0c9994e87a34e9d374d017087, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/8d94324f81244e4291ae886defcc0fc0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/909ff096419d48bdb07670703fb81c07] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=60.4 K 2024-04-14T11:15:34,781 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:34,782 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 1e08ac2e49d24d2a880faf3191a1734c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1713093332162 2024-04-14T11:15:34,783 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting d9864ac9d5db4600972d8564e4d0b7cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1713093332496 2024-04-14T11:15:34,783 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37534 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:34,783 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/B is initiating minor compaction (all files) 2024-04-14T11:15:34,783 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/B in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=20 2024-04-14T11:15:34,784 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/58e7ab9d98734338bdd1979179878189, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c92d4011bfd54413be80fe3e2303b473, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/af22681d8e844e83a572d5dfce7e79f4] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=36.7 K 2024-04-14T11:15:34,784 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 5dfa14b0c9994e87a34e9d374d017087, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1713093333140 2024-04-14T11:15:34,784 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 58e7ab9d98734338bdd1979179878189, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1713093333140 2024-04-14T11:15:34,785 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 8d94324f81244e4291ae886defcc0fc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1713093333794 2024-04-14T11:15:34,786 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c92d4011bfd54413be80fe3e2303b473, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1713093333794 2024-04-14T11:15:34,786 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 909ff096419d48bdb07670703fb81c07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1713093334443 2024-04-14T11:15:34,786 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting af22681d8e844e83a572d5dfce7e79f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1713093334443 2024-04-14T11:15:34,791 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:34,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-04-14T11:15:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:34,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/6f367ce6b47a48f4819ea24a08758ab8 is 50, key is test_row_0/A:col10/1713093334479/Put/seqid=0 2024-04-14T11:15:34,820 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#B#compaction#69 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:34,821 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/b5694b97dbe14280974f6102586b03d0 is 50, key is test_row_0/B:col10/1713093334448/Put/seqid=0 2024-04-14T11:15:34,824 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#C#compaction#70 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:34,825 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/d2bb722d630444b2a8cd7bcbad842170 is 50, key is test_row_0/C:col10/1713093334448/Put/seqid=0 2024-04-14T11:15:34,837 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-04-14T11:15:34,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:34,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:34,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:34,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(53): pid=21 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:34,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:34,850 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/d2bb722d630444b2a8cd7bcbad842170 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d2bb722d630444b2a8cd7bcbad842170 2024-04-14T11:15:34,865 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 5 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/C of 5407acadb5e9c9af78221a8f2d7e6785 into d2bb722d630444b2a8cd7bcbad842170(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:34,865 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:34,865 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/C, priority=11, startTime=1713093334606; duration=0sec 2024-04-14T11:15:34,865 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:34,865 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:C 2024-04-14T11:15:34,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093394819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093394819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093394866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093394866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093394867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093394970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093394970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093394975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093394975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093394975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=20 2024-04-14T11:15:34,993 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:34,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-04-14T11:15:34,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:34,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:34,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:34,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(53): pid=21 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:34,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:35,148 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-04-14T11:15:35,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:35,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:35,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:35,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(53): pid=21 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:35,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:35,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093395173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:35,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093395174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093395177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:35,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093395177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:35,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093395178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/6f367ce6b47a48f4819ea24a08758ab8 2024-04-14T11:15:35,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/a8e70c305c43457d95bcc73f1bcd8054 is 50, key is test_row_0/B:col10/1713093334479/Put/seqid=0 2024-04-14T11:15:35,252 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/b5694b97dbe14280974f6102586b03d0 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/b5694b97dbe14280974f6102586b03d0 2024-04-14T11:15:35,259 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/B of 5407acadb5e9c9af78221a8f2d7e6785 into b5694b97dbe14280974f6102586b03d0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:35,259 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:35,259 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/B, priority=13, startTime=1713093334605; duration=0sec 2024-04-14T11:15:35,260 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:35,260 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:B 2024-04-14T11:15:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=20 2024-04-14T11:15:35,303 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-04-14T11:15:35,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:35,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:35,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:35,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(53): pid=21 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:35,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:35,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:35,458 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-04-14T11:15:35,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:35,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:35,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:35,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(53): pid=21 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:35,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:35,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093395475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:35,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093395477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093395481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:35,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093395482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093395483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,613 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-04-14T11:15:35,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:35,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:35,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:35,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(53): pid=21 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:35,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:35,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/a8e70c305c43457d95bcc73f1bcd8054 2024-04-14T11:15:35,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/511790b2278b4c498cdb968ff9331e2b is 50, key is test_row_0/C:col10/1713093334479/Put/seqid=0 2024-04-14T11:15:35,771 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-04-14T11:15:35,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:35,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:35,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:35,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(53): pid=21 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:35,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=20 2024-04-14T11:15:35,926 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-04-14T11:15:35,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:35,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:35,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:35,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(53): pid=21 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:35,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:35,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093395979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093395980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093395984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093395987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:35,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093395990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:36,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/511790b2278b4c498cdb968ff9331e2b 2024-04-14T11:15:36,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/6f367ce6b47a48f4819ea24a08758ab8 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/6f367ce6b47a48f4819ea24a08758ab8 2024-04-14T11:15:36,081 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:36,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-04-14T11:15:36,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:36,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:36,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:36,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(53): pid=21 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:36,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:36,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/6f367ce6b47a48f4819ea24a08758ab8, entries=200, sequenceid=352, filesize=14.4 K 2024-04-14T11:15:36,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/a8e70c305c43457d95bcc73f1bcd8054 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/a8e70c305c43457d95bcc73f1bcd8054 2024-04-14T11:15:36,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/a8e70c305c43457d95bcc73f1bcd8054, entries=150, sequenceid=352, filesize=12.0 K 2024-04-14T11:15:36,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/511790b2278b4c498cdb968ff9331e2b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/511790b2278b4c498cdb968ff9331e2b 2024-04-14T11:15:36,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/511790b2278b4c498cdb968ff9331e2b, entries=150, sequenceid=352, filesize=12.0 K 2024-04-14T11:15:36,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=100.63 KB/103050 for 5407acadb5e9c9af78221a8f2d7e6785 in 1328ms, sequenceid=352, compaction requested=true 2024-04-14T11:15:36,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:36,119 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:15:36,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:15:36,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:36,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:15:36,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:36,120 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-04-14T11:15:36,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:15:36,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:36,121 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-04-14T11:15:36,121 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-04-14T11:15:36,121 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. because compaction request was cancelled 2024-04-14T11:15:36,121 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:B 2024-04-14T11:15:36,122 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-04-14T11:15:36,122 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54698 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:15:36,122 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/A is initiating minor compaction (all files) 2024-04-14T11:15:36,122 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/A in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:36,122 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e372b9d0582b4a6c97dc34d7758b6243, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d8f6bc1a89684f939e31c7e2de9ccdb3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/b882ad39904f48b3bb8ae6f6dcd145d8, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/6f367ce6b47a48f4819ea24a08758ab8] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=53.4 K 2024-04-14T11:15:36,123 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-04-14T11:15:36,123 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-04-14T11:15:36,123 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. because compaction request was cancelled 2024-04-14T11:15:36,123 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:C 2024-04-14T11:15:36,123 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting e372b9d0582b4a6c97dc34d7758b6243, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1713093333140 2024-04-14T11:15:36,124 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting d8f6bc1a89684f939e31c7e2de9ccdb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1713093333794 2024-04-14T11:15:36,124 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting b882ad39904f48b3bb8ae6f6dcd145d8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1713093334432 2024-04-14T11:15:36,125 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6f367ce6b47a48f4819ea24a08758ab8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1713093334476 2024-04-14T11:15:36,143 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#A#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:36,145 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/05894ea305e341cca653cdf56d41d1c5 is 50, key is test_row_0/A:col10/1713093334479/Put/seqid=0 2024-04-14T11:15:36,174 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/05894ea305e341cca653cdf56d41d1c5 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/05894ea305e341cca653cdf56d41d1c5 2024-04-14T11:15:36,187 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/A of 5407acadb5e9c9af78221a8f2d7e6785 into 05894ea305e341cca653cdf56d41d1c5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:36,187 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:36,187 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/A, priority=12, startTime=1713093336119; duration=0sec 2024-04-14T11:15:36,187 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:36,187 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:A 2024-04-14T11:15:36,236 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:36,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-04-14T11:15:36,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:36,237 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-04-14T11:15:36,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/93aa130b8586483d809402ac86519ef1 is 50, key is test_row_0/A:col10/1713093334826/Put/seqid=0 2024-04-14T11:15:36,649 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/93aa130b8586483d809402ac86519ef1 2024-04-14T11:15:36,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/094d1c403cb2473da01efda5de7d62ae is 50, key is test_row_0/B:col10/1713093334826/Put/seqid=0 2024-04-14T11:15:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=20 2024-04-14T11:15:36,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:36,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:37,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093397003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093397003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093397004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093397004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093397017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,075 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/094d1c403cb2473da01efda5de7d62ae 2024-04-14T11:15:37,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/e4efefd6149d4482a5e8327723df9c3f is 50, key is test_row_0/C:col10/1713093334826/Put/seqid=0 2024-04-14T11:15:37,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093397116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093397117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093397118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093397118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093397121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093397319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093397323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093397324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093397324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093397325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,494 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/e4efefd6149d4482a5e8327723df9c3f 2024-04-14T11:15:37,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/93aa130b8586483d809402ac86519ef1 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/93aa130b8586483d809402ac86519ef1 2024-04-14T11:15:37,510 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/93aa130b8586483d809402ac86519ef1, entries=150, sequenceid=373, filesize=12.0 K 2024-04-14T11:15:37,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/094d1c403cb2473da01efda5de7d62ae as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/094d1c403cb2473da01efda5de7d62ae 2024-04-14T11:15:37,519 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/094d1c403cb2473da01efda5de7d62ae, entries=150, sequenceid=373, filesize=12.0 K 2024-04-14T11:15:37,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/e4efefd6149d4482a5e8327723df9c3f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/e4efefd6149d4482a5e8327723df9c3f 2024-04-14T11:15:37,527 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/e4efefd6149d4482a5e8327723df9c3f, entries=150, sequenceid=373, filesize=12.0 K 2024-04-14T11:15:37,528 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(3035): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 5407acadb5e9c9af78221a8f2d7e6785 in 1291ms, sequenceid=373, compaction requested=true 2024-04-14T11:15:37,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:37,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:37,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=21 2024-04-14T11:15:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=21 2024-04-14T11:15:37,534 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=21, resume processing ppid=20 2024-04-14T11:15:37,534 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=21, ppid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8480 sec 2024-04-14T11:15:37,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees in 2.8570 sec 2024-04-14T11:15:37,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:37,627 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:37,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-04-14T11:15:37,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/84082eec732346fcab80a00a9c0fd4fc is 50, key is test_row_0/A:col10/1713093337625/Put/seqid=0 2024-04-14T11:15:37,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093397642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093397643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093397648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093397650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093397649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093397750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093397750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093397753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093397756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093397756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093397952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093397956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093397958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093397959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:37,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093397960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:38,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/84082eec732346fcab80a00a9c0fd4fc 2024-04-14T11:15:38,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/a9c57e0a63a449d0b8961f441f01d935 is 50, key is test_row_0/B:col10/1713093337625/Put/seqid=0 2024-04-14T11:15:38,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/a9c57e0a63a449d0b8961f441f01d935 2024-04-14T11:15:38,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/c767ef9c1a2c4827a8ab3ef41107d9ed is 50, key is test_row_0/C:col10/1713093337625/Put/seqid=0 2024-04-14T11:15:38,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:38,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093398257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:38,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093398261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:38,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093398261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:38,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:38,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093398263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:38,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:38,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093398265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:38,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/c767ef9c1a2c4827a8ab3ef41107d9ed 2024-04-14T11:15:38,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/84082eec732346fcab80a00a9c0fd4fc as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/84082eec732346fcab80a00a9c0fd4fc 2024-04-14T11:15:38,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/84082eec732346fcab80a00a9c0fd4fc, entries=150, sequenceid=393, filesize=12.0 K 2024-04-14T11:15:38,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/a9c57e0a63a449d0b8961f441f01d935 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/a9c57e0a63a449d0b8961f441f01d935 2024-04-14T11:15:38,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/a9c57e0a63a449d0b8961f441f01d935, entries=150, sequenceid=393, filesize=12.0 K 2024-04-14T11:15:38,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/c767ef9c1a2c4827a8ab3ef41107d9ed as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/c767ef9c1a2c4827a8ab3ef41107d9ed 2024-04-14T11:15:38,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/c767ef9c1a2c4827a8ab3ef41107d9ed, entries=150, sequenceid=393, filesize=12.0 K 2024-04-14T11:15:38,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 5407acadb5e9c9af78221a8f2d7e6785 in 988ms, sequenceid=393, compaction requested=true 2024-04-14T11:15:38,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:38,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:15:38,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:38,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:15:38,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:38,616 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:38,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:15:38,616 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:15:38,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:38,617 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37670 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:38,617 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/A is initiating minor compaction (all files) 2024-04-14T11:15:38,618 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/A in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:38,618 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/05894ea305e341cca653cdf56d41d1c5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/93aa130b8586483d809402ac86519ef1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/84082eec732346fcab80a00a9c0fd4fc] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=36.8 K 2024-04-14T11:15:38,618 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 05894ea305e341cca653cdf56d41d1c5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1713093334479 2024-04-14T11:15:38,619 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:15:38,619 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/B is initiating minor compaction (all files) 2024-04-14T11:15:38,619 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/B in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:38,619 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/b5694b97dbe14280974f6102586b03d0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/a8e70c305c43457d95bcc73f1bcd8054, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/094d1c403cb2473da01efda5de7d62ae, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/a9c57e0a63a449d0b8961f441f01d935] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=48.8 K 2024-04-14T11:15:38,619 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 93aa130b8586483d809402ac86519ef1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1713093334815 2024-04-14T11:15:38,620 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting b5694b97dbe14280974f6102586b03d0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1713093334443 2024-04-14T11:15:38,620 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 84082eec732346fcab80a00a9c0fd4fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1713093337002 2024-04-14T11:15:38,620 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting a8e70c305c43457d95bcc73f1bcd8054, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1713093334479 2024-04-14T11:15:38,621 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 094d1c403cb2473da01efda5de7d62ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1713093334815 2024-04-14T11:15:38,622 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting a9c57e0a63a449d0b8961f441f01d935, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1713093337002 2024-04-14T11:15:38,634 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#A#compaction#80 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:38,635 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/cac5e5910e614066a0ce92419ce131b4 is 50, key is test_row_0/A:col10/1713093337625/Put/seqid=0 2024-04-14T11:15:38,638 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#B#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:38,639 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/00603b192bdf4b81be57157af214dc19 is 50, key is test_row_0/B:col10/1713093337625/Put/seqid=0 2024-04-14T11:15:38,763 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:38,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-04-14T11:15:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:38,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/6c66551efc454be39b9d6bf06019cdf4 is 50, key is test_row_0/A:col10/1713093338761/Put/seqid=0 2024-04-14T11:15:38,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=20 2024-04-14T11:15:38,792 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 20 completed 2024-04-14T11:15:38,795 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:15:38,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=22, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees 2024-04-14T11:15:38,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093398790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:38,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093398791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:38,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2024-04-14T11:15:38,800 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=22, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:15:38,801 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=22, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:15:38,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:15:38,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093398794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:38,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093398799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:38,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093398809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:38,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2024-04-14T11:15:38,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:38,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093398901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093398901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:38,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:38,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093398903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:38,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:38,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093398903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:38,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:38,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093398912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:38,954 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:38,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2024-04-14T11:15:38,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:38,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:38,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:38,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] handler.RSProcedureHandler(53): pid=23 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:38,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=23 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:38,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=23 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:39,057 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/cac5e5910e614066a0ce92419ce131b4 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cac5e5910e614066a0ce92419ce131b4 2024-04-14T11:15:39,057 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/00603b192bdf4b81be57157af214dc19 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/00603b192bdf4b81be57157af214dc19 2024-04-14T11:15:39,068 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/B of 5407acadb5e9c9af78221a8f2d7e6785 into 00603b192bdf4b81be57157af214dc19(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:39,068 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/A of 5407acadb5e9c9af78221a8f2d7e6785 into cac5e5910e614066a0ce92419ce131b4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:39,068 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:39,068 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/A, priority=13, startTime=1713093338615; duration=0sec 2024-04-14T11:15:39,068 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:39,068 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:A 2024-04-14T11:15:39,068 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:15:39,068 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:39,068 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/B, priority=12, startTime=1713093338616; duration=0sec 2024-04-14T11:15:39,068 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:39,069 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:B 2024-04-14T11:15:39,071 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:15:39,071 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/C is initiating minor compaction (all files) 2024-04-14T11:15:39,071 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/C in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:39,071 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d2bb722d630444b2a8cd7bcbad842170, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/511790b2278b4c498cdb968ff9331e2b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/e4efefd6149d4482a5e8327723df9c3f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/c767ef9c1a2c4827a8ab3ef41107d9ed] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=48.7 K 2024-04-14T11:15:39,072 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting d2bb722d630444b2a8cd7bcbad842170, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1713093334443 2024-04-14T11:15:39,072 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 511790b2278b4c498cdb968ff9331e2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1713093334479 2024-04-14T11:15:39,078 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting e4efefd6149d4482a5e8327723df9c3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1713093334815 2024-04-14T11:15:39,079 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c767ef9c1a2c4827a8ab3ef41107d9ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1713093337002 2024-04-14T11:15:39,094 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#C#compaction#83 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:39,095 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/03f9128c67334e42b176beb1f4a9ae44 is 50, key is test_row_0/C:col10/1713093337625/Put/seqid=0 2024-04-14T11:15:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2024-04-14T11:15:39,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093399104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093399104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:39,108 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093399107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093399107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2024-04-14T11:15:39,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:39,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:39,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:39,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] handler.RSProcedureHandler(53): pid=23 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:39,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=23 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=23 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:39,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093399115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,130 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/03f9128c67334e42b176beb1f4a9ae44 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/03f9128c67334e42b176beb1f4a9ae44 2024-04-14T11:15:39,140 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/C of 5407acadb5e9c9af78221a8f2d7e6785 into 03f9128c67334e42b176beb1f4a9ae44(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:39,140 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:39,140 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/C, priority=12, startTime=1713093338616; duration=0sec 2024-04-14T11:15:39,140 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:39,140 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:C 2024-04-14T11:15:39,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/6c66551efc454be39b9d6bf06019cdf4 2024-04-14T11:15:39,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/bff7b47a39884902907941b22e552258 is 50, key is test_row_0/B:col10/1713093338761/Put/seqid=0 2024-04-14T11:15:39,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/bff7b47a39884902907941b22e552258 2024-04-14T11:15:39,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/facbe1c980d44ab0b364a78c9154f256 is 50, key is test_row_0/C:col10/1713093338761/Put/seqid=0 2024-04-14T11:15:39,261 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2024-04-14T11:15:39,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:39,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:39,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:39,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] handler.RSProcedureHandler(53): pid=23 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:39,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=23 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=23 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2024-04-14T11:15:39,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093399411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093399411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093399411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093399412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,415 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2024-04-14T11:15:39,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:39,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:39,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:39,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] handler.RSProcedureHandler(53): pid=23 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:39,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=23 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=23 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:39,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093399418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,569 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2024-04-14T11:15:39,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:39,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:39,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:39,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] handler.RSProcedureHandler(53): pid=23 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:39,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=23 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:39,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=23 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:39,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/facbe1c980d44ab0b364a78c9154f256 2024-04-14T11:15:39,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/6c66551efc454be39b9d6bf06019cdf4 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/6c66551efc454be39b9d6bf06019cdf4 2024-04-14T11:15:39,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/6c66551efc454be39b9d6bf06019cdf4, entries=200, sequenceid=411, filesize=14.4 K 2024-04-14T11:15:39,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/bff7b47a39884902907941b22e552258 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/bff7b47a39884902907941b22e552258 2024-04-14T11:15:39,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/bff7b47a39884902907941b22e552258, entries=150, sequenceid=411, filesize=12.0 K 2024-04-14T11:15:39,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/facbe1c980d44ab0b364a78c9154f256 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/facbe1c980d44ab0b364a78c9154f256 2024-04-14T11:15:39,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/facbe1c980d44ab0b364a78c9154f256, entries=150, sequenceid=411, filesize=12.0 K 2024-04-14T11:15:39,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 5407acadb5e9c9af78221a8f2d7e6785 in 890ms, sequenceid=411, compaction requested=false 2024-04-14T11:15:39,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:39,722 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2024-04-14T11:15:39,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:39,723 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-04-14T11:15:39,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/d9d1282edf374fa7a225d6da86eb07cf is 50, key is test_row_0/A:col10/1713093338779/Put/seqid=0 2024-04-14T11:15:39,735 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/d9d1282edf374fa7a225d6da86eb07cf 2024-04-14T11:15:39,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/9ba07f2bb32f4e05910f9708cd23501f is 50, key is test_row_0/B:col10/1713093338779/Put/seqid=0 2024-04-14T11:15:39,754 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/9ba07f2bb32f4e05910f9708cd23501f 2024-04-14T11:15:39,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/e92d7a79ab474ce7aabd985ac4a26dee is 50, key is test_row_0/C:col10/1713093338779/Put/seqid=0 2024-04-14T11:15:39,770 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/e92d7a79ab474ce7aabd985ac4a26dee 2024-04-14T11:15:39,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/d9d1282edf374fa7a225d6da86eb07cf as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d9d1282edf374fa7a225d6da86eb07cf 2024-04-14T11:15:39,788 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d9d1282edf374fa7a225d6da86eb07cf, entries=150, sequenceid=433, filesize=12.0 K 2024-04-14T11:15:39,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/9ba07f2bb32f4e05910f9708cd23501f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/9ba07f2bb32f4e05910f9708cd23501f 2024-04-14T11:15:39,799 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/9ba07f2bb32f4e05910f9708cd23501f, entries=150, sequenceid=433, filesize=12.0 K 2024-04-14T11:15:39,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/e92d7a79ab474ce7aabd985ac4a26dee as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/e92d7a79ab474ce7aabd985ac4a26dee 2024-04-14T11:15:39,808 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/e92d7a79ab474ce7aabd985ac4a26dee, entries=150, sequenceid=433, filesize=12.0 K 2024-04-14T11:15:39,810 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(3035): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for 5407acadb5e9c9af78221a8f2d7e6785 in 87ms, sequenceid=433, compaction requested=true 2024-04-14T11:15:39,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:39,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:39,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=23 2024-04-14T11:15:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=23 2024-04-14T11:15:39,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=23, resume processing ppid=22 2024-04-14T11:15:39,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=23, ppid=22, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0140 sec 2024-04-14T11:15:39,834 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=22, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees in 1.0360 sec 2024-04-14T11:15:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2024-04-14T11:15:39,905 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 22 completed 2024-04-14T11:15:39,907 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:15:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=24, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees 2024-04-14T11:15:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=24 2024-04-14T11:15:39,916 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=24, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:15:39,917 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=24, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:15:39,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:15:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:39,934 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:39,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-04-14T11:15:39,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/4c941a687be14eeabb526c6a40386578 is 50, key is test_row_0/A:col10/1713093339934/Put/seqid=0 2024-04-14T11:15:39,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/4c941a687be14eeabb526c6a40386578 2024-04-14T11:15:39,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/64106d0e214b4044bccf04afbfdfd6dc is 50, key is test_row_0/B:col10/1713093339934/Put/seqid=0 2024-04-14T11:15:39,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:39,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093399972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093399974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:39,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093399975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093399976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093399977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:39,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/64106d0e214b4044bccf04afbfdfd6dc 2024-04-14T11:15:40,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/bd66300bc2d04153be7e7a600064813a is 50, key is test_row_0/C:col10/1713093339934/Put/seqid=0 2024-04-14T11:15:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=24 2024-04-14T11:15:40,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/bd66300bc2d04153be7e7a600064813a 2024-04-14T11:15:40,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/4c941a687be14eeabb526c6a40386578 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/4c941a687be14eeabb526c6a40386578 2024-04-14T11:15:40,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/4c941a687be14eeabb526c6a40386578, entries=250, sequenceid=446, filesize=16.8 K 2024-04-14T11:15:40,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/64106d0e214b4044bccf04afbfdfd6dc as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/64106d0e214b4044bccf04afbfdfd6dc 2024-04-14T11:15:40,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/64106d0e214b4044bccf04afbfdfd6dc, entries=150, sequenceid=446, filesize=12.0 K 2024-04-14T11:15:40,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/bd66300bc2d04153be7e7a600064813a as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/bd66300bc2d04153be7e7a600064813a 2024-04-14T11:15:40,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/bd66300bc2d04153be7e7a600064813a, entries=150, sequenceid=446, filesize=12.0 K 2024-04-14T11:15:40,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 5407acadb5e9c9af78221a8f2d7e6785 in 133ms, sequenceid=446, compaction requested=true 2024-04-14T11:15:40,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:40,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:15:40,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:40,068 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:15:40,068 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:15:40,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:15:40,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:40,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:15:40,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:40,069 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=25 2024-04-14T11:15:40,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:40,070 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-04-14T11:15:40,072 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:15:40,072 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/B is initiating minor compaction (all files) 2024-04-14T11:15:40,072 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/B in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:40,072 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/00603b192bdf4b81be57157af214dc19, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/bff7b47a39884902907941b22e552258, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/9ba07f2bb32f4e05910f9708cd23501f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/64106d0e214b4044bccf04afbfdfd6dc] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=48.9 K 2024-04-14T11:15:40,074 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57376 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:15:40,074 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/A is initiating minor compaction (all files) 2024-04-14T11:15:40,074 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/A in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:40,074 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cac5e5910e614066a0ce92419ce131b4, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/6c66551efc454be39b9d6bf06019cdf4, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d9d1282edf374fa7a225d6da86eb07cf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/4c941a687be14eeabb526c6a40386578] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=56.0 K 2024-04-14T11:15:40,075 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 00603b192bdf4b81be57157af214dc19, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1713093337002 2024-04-14T11:15:40,075 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting cac5e5910e614066a0ce92419ce131b4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1713093337002 2024-04-14T11:15:40,076 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting bff7b47a39884902907941b22e552258, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1713093337646 2024-04-14T11:15:40,076 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6c66551efc454be39b9d6bf06019cdf4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1713093337640 2024-04-14T11:15:40,077 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 9ba07f2bb32f4e05910f9708cd23501f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1713093338779 2024-04-14T11:15:40,077 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting d9d1282edf374fa7a225d6da86eb07cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1713093338779 2024-04-14T11:15:40,077 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 64106d0e214b4044bccf04afbfdfd6dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1713093339917 2024-04-14T11:15:40,078 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4c941a687be14eeabb526c6a40386578, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1713093339917 2024-04-14T11:15:40,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/725b338633a54c0484a33b1619d43adc is 50, key is test_row_0/A:col10/1713093339966/Put/seqid=0 2024-04-14T11:15:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:40,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:40,098 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#B#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:40,099 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/173a10f2885a4787bb9f2f2b849bf6ba is 50, key is test_row_0/B:col10/1713093339934/Put/seqid=0 2024-04-14T11:15:40,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093400094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,103 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#A#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:40,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093400096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,104 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/2b5c1c25d4f748689299eb5815b51dd7 is 50, key is test_row_0/A:col10/1713093339934/Put/seqid=0 2024-04-14T11:15:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093400100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093400103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093400103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,112 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/725b338633a54c0484a33b1619d43adc 2024-04-14T11:15:40,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/b01dbfd5c2bb487783ec501e0a0fa30c is 50, key is test_row_0/B:col10/1713093339966/Put/seqid=0 2024-04-14T11:15:40,145 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/173a10f2885a4787bb9f2f2b849bf6ba as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/173a10f2885a4787bb9f2f2b849bf6ba 2024-04-14T11:15:40,146 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/2b5c1c25d4f748689299eb5815b51dd7 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/2b5c1c25d4f748689299eb5815b51dd7 2024-04-14T11:15:40,149 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/b01dbfd5c2bb487783ec501e0a0fa30c 2024-04-14T11:15:40,159 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/B of 5407acadb5e9c9af78221a8f2d7e6785 into 173a10f2885a4787bb9f2f2b849bf6ba(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:40,159 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/A of 5407acadb5e9c9af78221a8f2d7e6785 into 2b5c1c25d4f748689299eb5815b51dd7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:40,159 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:40,159 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/A, priority=12, startTime=1713093340068; duration=0sec 2024-04-14T11:15:40,159 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:40,159 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:A 2024-04-14T11:15:40,159 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:15:40,159 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:40,159 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/B, priority=12, startTime=1713093340068; duration=0sec 2024-04-14T11:15:40,160 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:40,160 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:B 2024-04-14T11:15:40,163 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:15:40,163 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/C is initiating minor compaction (all files) 2024-04-14T11:15:40,163 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/C in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:40,163 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/03f9128c67334e42b176beb1f4a9ae44, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/facbe1c980d44ab0b364a78c9154f256, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/e92d7a79ab474ce7aabd985ac4a26dee, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/bd66300bc2d04153be7e7a600064813a] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=48.8 K 2024-04-14T11:15:40,164 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 03f9128c67334e42b176beb1f4a9ae44, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1713093337002 2024-04-14T11:15:40,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/1ce724858c6f4ec9a8f0e3b62be37032 is 50, key is test_row_0/C:col10/1713093339966/Put/seqid=0 2024-04-14T11:15:40,164 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting facbe1c980d44ab0b364a78c9154f256, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1713093337646 2024-04-14T11:15:40,165 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting e92d7a79ab474ce7aabd985ac4a26dee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1713093338779 2024-04-14T11:15:40,166 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting bd66300bc2d04153be7e7a600064813a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1713093339917 2024-04-14T11:15:40,173 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/1ce724858c6f4ec9a8f0e3b62be37032 2024-04-14T11:15:40,178 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#C#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:40,178 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/be1c4ddc3b6243199128c3c8f048c155 is 50, key is test_row_0/C:col10/1713093339934/Put/seqid=0 2024-04-14T11:15:40,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/725b338633a54c0484a33b1619d43adc as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/725b338633a54c0484a33b1619d43adc 2024-04-14T11:15:40,190 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/725b338633a54c0484a33b1619d43adc, entries=150, sequenceid=470, filesize=12.0 K 2024-04-14T11:15:40,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/b01dbfd5c2bb487783ec501e0a0fa30c as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/b01dbfd5c2bb487783ec501e0a0fa30c 2024-04-14T11:15:40,203 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/b01dbfd5c2bb487783ec501e0a0fa30c, entries=150, sequenceid=470, filesize=12.0 K 2024-04-14T11:15:40,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/1ce724858c6f4ec9a8f0e3b62be37032 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/1ce724858c6f4ec9a8f0e3b62be37032 2024-04-14T11:15:40,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093400204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093400205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093400206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093400206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093400207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,214 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/1ce724858c6f4ec9a8f0e3b62be37032, entries=150, sequenceid=470, filesize=12.0 K 2024-04-14T11:15:40,215 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(3035): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 5407acadb5e9c9af78221a8f2d7e6785 in 145ms, sequenceid=470, compaction requested=false 2024-04-14T11:15:40,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:40,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:40,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=25 2024-04-14T11:15:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=25 2024-04-14T11:15:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=24 2024-04-14T11:15:40,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=25, resume processing ppid=24 2024-04-14T11:15:40,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=25, ppid=24, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 299 msec 2024-04-14T11:15:40,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=24, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees in 314 msec 2024-04-14T11:15:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:40,413 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:40,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-04-14T11:15:40,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/cf2f22a645ab4a4196134ddc3e753ed5 is 50, key is test_row_0/A:col10/1713093340101/Put/seqid=0 2024-04-14T11:15:40,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093400431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093400434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093400434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093400436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093400436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=24 2024-04-14T11:15:40,519 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 24 completed 2024-04-14T11:15:40,521 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:15:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=26, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees 2024-04-14T11:15:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2024-04-14T11:15:40,525 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=26, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:15:40,526 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=26, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:15:40,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:15:40,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093400538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093400548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093400548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093400548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093400549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,597 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/be1c4ddc3b6243199128c3c8f048c155 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/be1c4ddc3b6243199128c3c8f048c155 2024-04-14T11:15:40,605 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/C of 5407acadb5e9c9af78221a8f2d7e6785 into be1c4ddc3b6243199128c3c8f048c155(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:40,605 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:40,605 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/C, priority=12, startTime=1713093340068; duration=0sec 2024-04-14T11:15:40,605 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:40,605 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:C 2024-04-14T11:15:40,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2024-04-14T11:15:40,679 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-04-14T11:15:40,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:40,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:40,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:40,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(53): pid=27 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:40,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:40,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093400750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093400750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093400751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093400751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093400751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2024-04-14T11:15:40,832 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-04-14T11:15:40,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:40,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:40,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:40,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(53): pid=27 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:40,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:40,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/cf2f22a645ab4a4196134ddc3e753ed5 2024-04-14T11:15:40,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/3fc6d7387d5247b598dea7c36382127d is 50, key is test_row_0/B:col10/1713093340101/Put/seqid=0 2024-04-14T11:15:40,985 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:40,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-04-14T11:15:40,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:40,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:40,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:40,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(53): pid=27 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:40,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:41,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093401052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:41,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093401053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:41,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093401055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:41,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093401054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:41,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093401055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:41,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2024-04-14T11:15:41,141 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:41,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-04-14T11:15:41,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:41,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:41,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:41,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(53): pid=27 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:41,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:41,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:41,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/3fc6d7387d5247b598dea7c36382127d 2024-04-14T11:15:41,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/46202302ce4a46f0bdfad9a67519caf7 is 50, key is test_row_0/C:col10/1713093340101/Put/seqid=0 2024-04-14T11:15:41,295 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:41,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-04-14T11:15:41,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:41,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:41,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:41,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(53): pid=27 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:41,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:41,448 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:41,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-04-14T11:15:41,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:41,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:41,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:41,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(53): pid=27 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:41,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:41,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093401557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:41,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093401557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:41,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093401558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:41,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093401559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:41,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093401560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:41,601 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:41,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-04-14T11:15:41,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:41,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:41,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:41,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(53): pid=27 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:41,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:41,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2024-04-14T11:15:41,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/46202302ce4a46f0bdfad9a67519caf7 2024-04-14T11:15:41,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/cf2f22a645ab4a4196134ddc3e753ed5 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cf2f22a645ab4a4196134ddc3e753ed5 2024-04-14T11:15:41,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cf2f22a645ab4a4196134ddc3e753ed5, entries=250, sequenceid=488, filesize=16.8 K 2024-04-14T11:15:41,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/3fc6d7387d5247b598dea7c36382127d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/3fc6d7387d5247b598dea7c36382127d 2024-04-14T11:15:41,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/3fc6d7387d5247b598dea7c36382127d, entries=150, sequenceid=488, filesize=12.0 K 2024-04-14T11:15:41,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/46202302ce4a46f0bdfad9a67519caf7 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/46202302ce4a46f0bdfad9a67519caf7 2024-04-14T11:15:41,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/46202302ce4a46f0bdfad9a67519caf7, entries=150, sequenceid=488, filesize=12.0 K 2024-04-14T11:15:41,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 5407acadb5e9c9af78221a8f2d7e6785 in 1292ms, sequenceid=488, compaction requested=true 2024-04-14T11:15:41,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:41,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:15:41,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:41,705 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:41,705 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:41,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:15:41,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:41,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:15:41,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:41,707 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42788 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:41,707 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/A is initiating minor compaction (all files) 2024-04-14T11:15:41,707 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/A in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:41,707 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/2b5c1c25d4f748689299eb5815b51dd7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/725b338633a54c0484a33b1619d43adc, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cf2f22a645ab4a4196134ddc3e753ed5] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=41.8 K 2024-04-14T11:15:41,707 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37908 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:41,708 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/B is initiating minor compaction (all files) 2024-04-14T11:15:41,708 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/B in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:41,708 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/173a10f2885a4787bb9f2f2b849bf6ba, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/b01dbfd5c2bb487783ec501e0a0fa30c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/3fc6d7387d5247b598dea7c36382127d] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=37.0 K 2024-04-14T11:15:41,708 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2b5c1c25d4f748689299eb5815b51dd7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1713093339917 2024-04-14T11:15:41,708 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 173a10f2885a4787bb9f2f2b849bf6ba, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1713093339917 2024-04-14T11:15:41,709 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 725b338633a54c0484a33b1619d43adc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1713093339966 2024-04-14T11:15:41,709 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting b01dbfd5c2bb487783ec501e0a0fa30c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1713093339966 2024-04-14T11:15:41,709 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting cf2f22a645ab4a4196134ddc3e753ed5, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1713093340092 2024-04-14T11:15:41,710 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 3fc6d7387d5247b598dea7c36382127d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1713093340101 2024-04-14T11:15:41,726 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#A#compaction#101 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:41,727 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/cb8c0c7bea83417da9774e93c860bcd0 is 50, key is test_row_0/A:col10/1713093340101/Put/seqid=0 2024-04-14T11:15:41,729 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#B#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:41,730 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/f363bb1e37c54a2ca472e9e64853409e is 50, key is test_row_0/B:col10/1713093340101/Put/seqid=0 2024-04-14T11:15:41,754 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:41,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-04-14T11:15:41,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:41,756 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-04-14T11:15:41,759 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/cb8c0c7bea83417da9774e93c860bcd0 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cb8c0c7bea83417da9774e93c860bcd0 2024-04-14T11:15:41,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/e3c3d26fc32445bdae7dacd7e6500c61 is 50, key is test_row_0/A:col10/1713093340433/Put/seqid=0 2024-04-14T11:15:41,773 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/A of 5407acadb5e9c9af78221a8f2d7e6785 into cb8c0c7bea83417da9774e93c860bcd0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:41,773 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:41,773 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/A, priority=13, startTime=1713093341705; duration=0sec 2024-04-14T11:15:41,773 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:41,773 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:A 2024-04-14T11:15:41,773 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:41,775 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37874 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:41,775 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/C is initiating minor compaction (all files) 2024-04-14T11:15:41,775 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/C in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:41,776 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/be1c4ddc3b6243199128c3c8f048c155, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/1ce724858c6f4ec9a8f0e3b62be37032, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/46202302ce4a46f0bdfad9a67519caf7] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=37.0 K 2024-04-14T11:15:41,776 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting be1c4ddc3b6243199128c3c8f048c155, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1713093339917 2024-04-14T11:15:41,777 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1ce724858c6f4ec9a8f0e3b62be37032, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1713093339966 2024-04-14T11:15:41,777 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 46202302ce4a46f0bdfad9a67519caf7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1713093340101 2024-04-14T11:15:41,795 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#C#compaction#104 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:41,795 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/2b04186d45c34ca3b2b96f54e298bd38 is 50, key is test_row_0/C:col10/1713093340101/Put/seqid=0 2024-04-14T11:15:42,151 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/f363bb1e37c54a2ca472e9e64853409e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/f363bb1e37c54a2ca472e9e64853409e 2024-04-14T11:15:42,164 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/B of 5407acadb5e9c9af78221a8f2d7e6785 into f363bb1e37c54a2ca472e9e64853409e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:42,164 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:42,164 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/B, priority=13, startTime=1713093341705; duration=0sec 2024-04-14T11:15:42,164 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:42,164 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:B 2024-04-14T11:15:42,199 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/e3c3d26fc32445bdae7dacd7e6500c61 2024-04-14T11:15:42,215 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/2b04186d45c34ca3b2b96f54e298bd38 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/2b04186d45c34ca3b2b96f54e298bd38 2024-04-14T11:15:42,224 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/C of 5407acadb5e9c9af78221a8f2d7e6785 into 2b04186d45c34ca3b2b96f54e298bd38(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:42,224 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:42,225 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/C, priority=13, startTime=1713093341706; duration=0sec 2024-04-14T11:15:42,225 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:42,225 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:C 2024-04-14T11:15:42,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/aff142545c904468a3d9f10a5320b707 is 50, key is test_row_0/B:col10/1713093340433/Put/seqid=0 2024-04-14T11:15:42,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:42,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:42,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093402579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:42,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:42,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093402579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:42,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093402582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:42,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093402583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:42,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093402586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2024-04-14T11:15:42,639 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/aff142545c904468a3d9f10a5320b707 2024-04-14T11:15:42,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/f180c9cf91364798907b0b42dcb301e8 is 50, key is test_row_0/C:col10/1713093340433/Put/seqid=0 2024-04-14T11:15:42,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093402684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:42,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093402686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:42,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093402686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:42,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093402688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:42,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093402690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:42,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093402888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:42,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093402889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:42,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093402890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:42,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093402891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:42,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:42,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093402893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,061 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/f180c9cf91364798907b0b42dcb301e8 2024-04-14T11:15:43,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/e3c3d26fc32445bdae7dacd7e6500c61 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e3c3d26fc32445bdae7dacd7e6500c61 2024-04-14T11:15:43,075 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e3c3d26fc32445bdae7dacd7e6500c61, entries=150, sequenceid=510, filesize=12.0 K 2024-04-14T11:15:43,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/aff142545c904468a3d9f10a5320b707 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/aff142545c904468a3d9f10a5320b707 2024-04-14T11:15:43,089 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/aff142545c904468a3d9f10a5320b707, entries=150, sequenceid=510, filesize=12.0 K 2024-04-14T11:15:43,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/f180c9cf91364798907b0b42dcb301e8 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/f180c9cf91364798907b0b42dcb301e8 2024-04-14T11:15:43,098 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/f180c9cf91364798907b0b42dcb301e8, entries=150, sequenceid=510, filesize=12.0 K 2024-04-14T11:15:43,100 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(3035): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 5407acadb5e9c9af78221a8f2d7e6785 in 1344ms, sequenceid=510, compaction requested=false 2024-04-14T11:15:43,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:43,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:43,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=27 2024-04-14T11:15:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=27 2024-04-14T11:15:43,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=27, resume processing ppid=26 2024-04-14T11:15:43,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=27, ppid=26, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5750 sec 2024-04-14T11:15:43,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=26, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees in 2.5840 sec 2024-04-14T11:15:43,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:43,194 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:43,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-04-14T11:15:43,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/8c5de8a7b8be46cca33e270e79954d3b is 50, key is test_row_0/A:col10/1713093343194/Put/seqid=0 2024-04-14T11:15:43,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093403207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093403208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093403209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093403210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093403210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093403311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093403312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093403312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093403313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093403313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093403515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093403517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093403517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093403517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093403518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/8c5de8a7b8be46cca33e270e79954d3b 2024-04-14T11:15:43,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/ba7f1b8206114905b23eb506c3e32b99 is 50, key is test_row_0/B:col10/1713093343194/Put/seqid=0 2024-04-14T11:15:43,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093403819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093403819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093403820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093403821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:43,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:43,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093403821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/ba7f1b8206114905b23eb506c3e32b99 2024-04-14T11:15:44,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/b8b09bd80daa45618b899026d933ca7f is 50, key is test_row_0/C:col10/1713093343194/Put/seqid=0 2024-04-14T11:15:44,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/b8b09bd80daa45618b899026d933ca7f 2024-04-14T11:15:44,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/8c5de8a7b8be46cca33e270e79954d3b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/8c5de8a7b8be46cca33e270e79954d3b 2024-04-14T11:15:44,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/8c5de8a7b8be46cca33e270e79954d3b, entries=200, sequenceid=532, filesize=14.4 K 2024-04-14T11:15:44,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/ba7f1b8206114905b23eb506c3e32b99 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/ba7f1b8206114905b23eb506c3e32b99 2024-04-14T11:15:44,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/ba7f1b8206114905b23eb506c3e32b99, entries=150, sequenceid=532, filesize=12.0 K 2024-04-14T11:15:44,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/b8b09bd80daa45618b899026d933ca7f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/b8b09bd80daa45618b899026d933ca7f 2024-04-14T11:15:44,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/b8b09bd80daa45618b899026d933ca7f, entries=150, sequenceid=532, filesize=12.0 K 2024-04-14T11:15:44,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5407acadb5e9c9af78221a8f2d7e6785 in 879ms, sequenceid=532, compaction requested=true 2024-04-14T11:15:44,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:44,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:15:44,074 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:44,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:44,074 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:44,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:15:44,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:44,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:15:44,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:44,077 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40450 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:44,077 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/A is initiating minor compaction (all files) 2024-04-14T11:15:44,077 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/A in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:44,077 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38010 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:44,077 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cb8c0c7bea83417da9774e93c860bcd0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e3c3d26fc32445bdae7dacd7e6500c61, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/8c5de8a7b8be46cca33e270e79954d3b] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=39.5 K 2024-04-14T11:15:44,077 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/B is initiating minor compaction (all files) 2024-04-14T11:15:44,077 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/B in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:44,077 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/f363bb1e37c54a2ca472e9e64853409e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/aff142545c904468a3d9f10a5320b707, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/ba7f1b8206114905b23eb506c3e32b99] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=37.1 K 2024-04-14T11:15:44,078 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting cb8c0c7bea83417da9774e93c860bcd0, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1713093340101 2024-04-14T11:15:44,078 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting f363bb1e37c54a2ca472e9e64853409e, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1713093340101 2024-04-14T11:15:44,078 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting e3c3d26fc32445bdae7dacd7e6500c61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1713093340428 2024-04-14T11:15:44,079 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting aff142545c904468a3d9f10a5320b707, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1713093340428 2024-04-14T11:15:44,079 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8c5de8a7b8be46cca33e270e79954d3b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1713093342574 2024-04-14T11:15:44,079 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting ba7f1b8206114905b23eb506c3e32b99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1713093342574 2024-04-14T11:15:44,096 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#A#compaction#110 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:44,097 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/449550338a6d4d45ae074badb604473e is 50, key is test_row_0/A:col10/1713093343194/Put/seqid=0 2024-04-14T11:15:44,100 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#B#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:44,101 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/83f5d9251b6b43e5889399e33e5bede3 is 50, key is test_row_0/B:col10/1713093343194/Put/seqid=0 2024-04-14T11:15:44,132 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/83f5d9251b6b43e5889399e33e5bede3 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/83f5d9251b6b43e5889399e33e5bede3 2024-04-14T11:15:44,141 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/B of 5407acadb5e9c9af78221a8f2d7e6785 into 83f5d9251b6b43e5889399e33e5bede3(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:44,141 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:44,142 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/B, priority=13, startTime=1713093344074; duration=0sec 2024-04-14T11:15:44,142 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:44,142 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:B 2024-04-14T11:15:44,142 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:44,144 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37976 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:44,144 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/C is initiating minor compaction (all files) 2024-04-14T11:15:44,144 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/C in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:44,144 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/2b04186d45c34ca3b2b96f54e298bd38, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/f180c9cf91364798907b0b42dcb301e8, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/b8b09bd80daa45618b899026d933ca7f] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=37.1 K 2024-04-14T11:15:44,145 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 2b04186d45c34ca3b2b96f54e298bd38, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1713093340101 2024-04-14T11:15:44,145 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting f180c9cf91364798907b0b42dcb301e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1713093340428 2024-04-14T11:15:44,146 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting b8b09bd80daa45618b899026d933ca7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1713093342574 2024-04-14T11:15:44,155 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#C#compaction#112 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:44,156 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/7aab326791d6413cb3c0b004084936c4 is 50, key is test_row_0/C:col10/1713093343194/Put/seqid=0 2024-04-14T11:15:44,179 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/7aab326791d6413cb3c0b004084936c4 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/7aab326791d6413cb3c0b004084936c4 2024-04-14T11:15:44,188 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/C of 5407acadb5e9c9af78221a8f2d7e6785 into 7aab326791d6413cb3c0b004084936c4(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:44,188 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:44,188 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/C, priority=13, startTime=1713093344074; duration=0sec 2024-04-14T11:15:44,189 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:44,189 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:C 2024-04-14T11:15:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:44,324 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:44,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-04-14T11:15:44,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/264948e66a7c419bbf4c0df231c4ac25 is 50, key is test_row_0/A:col10/1713093344325/Put/seqid=0 2024-04-14T11:15:44,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093404336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093404337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093404338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093404338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093404339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093404440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093404440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093404443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093404443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093404444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,524 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/449550338a6d4d45ae074badb604473e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/449550338a6d4d45ae074badb604473e 2024-04-14T11:15:44,532 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/A of 5407acadb5e9c9af78221a8f2d7e6785 into 449550338a6d4d45ae074badb604473e(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:44,532 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:44,532 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/A, priority=13, startTime=1713093344074; duration=0sec 2024-04-14T11:15:44,532 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:44,532 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:A 2024-04-14T11:15:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2024-04-14T11:15:44,632 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 26 completed 2024-04-14T11:15:44,633 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:15:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=28, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees 2024-04-14T11:15:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=28 2024-04-14T11:15:44,637 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=28, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:15:44,638 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=28, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:15:44,638 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:15:44,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093404643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093404643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093404644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093404645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093404648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=28 2024-04-14T11:15:44,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/264948e66a7c419bbf4c0df231c4ac25 2024-04-14T11:15:44,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/ffc38569b95446a8af30f67e0d864e34 is 50, key is test_row_0/B:col10/1713093344325/Put/seqid=0 2024-04-14T11:15:44,791 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-04-14T11:15:44,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:44,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:44,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:44,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(53): pid=29 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:44,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:44,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:44,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=28 2024-04-14T11:15:44,944 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-04-14T11:15:44,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:44,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:44,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:44,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(53): pid=29 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:44,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:44,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093404945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:44,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093404947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093404949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093404949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:44,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:44,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093404952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:45,097 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:45,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-04-14T11:15:45,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:45,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. as already flushing 2024-04-14T11:15:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:45,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(53): pid=29 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:45,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/ffc38569b95446a8af30f67e0d864e34 2024-04-14T11:15:45,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/37a901cad1ac415baa53c194537a9e9c is 50, key is test_row_0/C:col10/1713093344325/Put/seqid=0 2024-04-14T11:15:45,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/37a901cad1ac415baa53c194537a9e9c 2024-04-14T11:15:45,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/264948e66a7c419bbf4c0df231c4ac25 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/264948e66a7c419bbf4c0df231c4ac25 2024-04-14T11:15:45,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/264948e66a7c419bbf4c0df231c4ac25, entries=200, sequenceid=553, filesize=14.4 K 2024-04-14T11:15:45,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/ffc38569b95446a8af30f67e0d864e34 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/ffc38569b95446a8af30f67e0d864e34 2024-04-14T11:15:45,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/ffc38569b95446a8af30f67e0d864e34, entries=150, sequenceid=553, filesize=12.0 K 2024-04-14T11:15:45,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/37a901cad1ac415baa53c194537a9e9c as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/37a901cad1ac415baa53c194537a9e9c 2024-04-14T11:15:45,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/37a901cad1ac415baa53c194537a9e9c, entries=150, sequenceid=553, filesize=12.0 K 2024-04-14T11:15:45,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 5407acadb5e9c9af78221a8f2d7e6785 in 907ms, sequenceid=553, compaction requested=false 2024-04-14T11:15:45,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:45,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=28 2024-04-14T11:15:45,254 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:45,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-04-14T11:15:45,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:45,255 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-04-14T11:15:45,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/eb3661eb132d46daafd0b8c955a7b25d is 50, key is test_row_0/A:col10/1713093344336/Put/seqid=0 2024-04-14T11:15:45,286 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/eb3661eb132d46daafd0b8c955a7b25d 2024-04-14T11:15:45,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/e635248d188a4cab98f972f10938c6c1 is 50, key is test_row_0/B:col10/1713093344336/Put/seqid=0 2024-04-14T11:15:45,308 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/e635248d188a4cab98f972f10938c6c1 2024-04-14T11:15:45,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/90a550f867004f019721b35ca7841746 is 50, key is test_row_0/C:col10/1713093344336/Put/seqid=0 2024-04-14T11:15:45,337 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/90a550f867004f019721b35ca7841746 2024-04-14T11:15:45,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/eb3661eb132d46daafd0b8c955a7b25d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/eb3661eb132d46daafd0b8c955a7b25d 2024-04-14T11:15:45,358 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/eb3661eb132d46daafd0b8c955a7b25d, entries=150, sequenceid=572, filesize=12.0 K 2024-04-14T11:15:45,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/e635248d188a4cab98f972f10938c6c1 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/e635248d188a4cab98f972f10938c6c1 2024-04-14T11:15:45,375 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/e635248d188a4cab98f972f10938c6c1, entries=150, sequenceid=572, filesize=12.0 K 2024-04-14T11:15:45,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/90a550f867004f019721b35ca7841746 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/90a550f867004f019721b35ca7841746 2024-04-14T11:15:45,386 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/90a550f867004f019721b35ca7841746, entries=150, sequenceid=572, filesize=12.0 K 2024-04-14T11:15:45,387 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(3035): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=0 B/0 for 5407acadb5e9c9af78221a8f2d7e6785 in 132ms, sequenceid=572, compaction requested=true 2024-04-14T11:15:45,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:45,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:45,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=29 2024-04-14T11:15:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=29 2024-04-14T11:15:45,393 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=29, resume processing ppid=28 2024-04-14T11:15:45,393 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=29, ppid=28, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 753 msec 2024-04-14T11:15:45,395 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=28, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees in 760 msec 2024-04-14T11:15:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:45,459 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:45,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-04-14T11:15:45,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/3162da97f5c843c0ac163effbb012aba is 50, key is test_row_0/A:col10/1713093345459/Put/seqid=0 2024-04-14T11:15:45,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093405483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:45,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093405484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:45,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093405486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:45,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093405488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:45,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093405488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:45,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=585 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/3162da97f5c843c0ac163effbb012aba 2024-04-14T11:15:45,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/3dce0c80eb664c5db501c936c875bdb9 is 50, key is test_row_0/B:col10/1713093345459/Put/seqid=0 2024-04-14T11:15:45,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=585 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/3dce0c80eb664c5db501c936c875bdb9 2024-04-14T11:15:45,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/747af3e3244b48b5a9847da90b59fd78 is 50, key is test_row_0/C:col10/1713093345459/Put/seqid=0 2024-04-14T11:15:45,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=585 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/747af3e3244b48b5a9847da90b59fd78 2024-04-14T11:15:45,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/3162da97f5c843c0ac163effbb012aba as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/3162da97f5c843c0ac163effbb012aba 2024-04-14T11:15:45,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/3162da97f5c843c0ac163effbb012aba, entries=300, sequenceid=585, filesize=19.1 K 2024-04-14T11:15:45,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/3dce0c80eb664c5db501c936c875bdb9 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/3dce0c80eb664c5db501c936c875bdb9 2024-04-14T11:15:45,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55636 deadline: 1713093405589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:45,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55604 deadline: 1713093405590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:45,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55618 deadline: 1713093405592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:45,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:45,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55630 deadline: 1713093405592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:45,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:45,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:55626 deadline: 1713093405593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:45,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/3dce0c80eb664c5db501c936c875bdb9, entries=150, sequenceid=585, filesize=12.0 K 2024-04-14T11:15:45,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/747af3e3244b48b5a9847da90b59fd78 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/747af3e3244b48b5a9847da90b59fd78 2024-04-14T11:15:45,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/747af3e3244b48b5a9847da90b59fd78, entries=150, sequenceid=585, filesize=12.0 K 2024-04-14T11:15:45,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 5407acadb5e9c9af78221a8f2d7e6785 in 159ms, sequenceid=585, compaction requested=true 2024-04-14T11:15:45,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:45,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:15:45,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:45,620 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:15:45,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:15:45,620 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:15:45,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:45,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:15:45,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:45,623 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 60156 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:15:45,623 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/A is initiating minor compaction (all files) 2024-04-14T11:15:45,623 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/A in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:45,623 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/449550338a6d4d45ae074badb604473e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/264948e66a7c419bbf4c0df231c4ac25, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/eb3661eb132d46daafd0b8c955a7b25d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/3162da97f5c843c0ac163effbb012aba] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=58.7 K 2024-04-14T11:15:45,624 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:15:45,624 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 5407acadb5e9c9af78221a8f2d7e6785/B is initiating minor compaction (all files) 2024-04-14T11:15:45,624 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 5407acadb5e9c9af78221a8f2d7e6785/B in TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:45,624 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/83f5d9251b6b43e5889399e33e5bede3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/ffc38569b95446a8af30f67e0d864e34, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/e635248d188a4cab98f972f10938c6c1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/3dce0c80eb664c5db501c936c875bdb9] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp, totalSize=49.2 K 2024-04-14T11:15:45,625 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 449550338a6d4d45ae074badb604473e, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1713093342574 2024-04-14T11:15:45,625 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 83f5d9251b6b43e5889399e33e5bede3, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1713093342574 2024-04-14T11:15:45,625 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 264948e66a7c419bbf4c0df231c4ac25, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=553, earliestPutTs=1713093343206 2024-04-14T11:15:45,626 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting ffc38569b95446a8af30f67e0d864e34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=553, earliestPutTs=1713093343206 2024-04-14T11:15:45,626 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting eb3661eb132d46daafd0b8c955a7b25d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1713093344333 2024-04-14T11:15:45,627 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting e635248d188a4cab98f972f10938c6c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1713093344333 2024-04-14T11:15:45,627 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 3dce0c80eb664c5db501c936c875bdb9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1713093345457 2024-04-14T11:15:45,628 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3162da97f5c843c0ac163effbb012aba, keycount=300, bloomtype=ROW, size=19.1 K, encoding=NONE, compression=NONE, seqNum=585, earliestPutTs=1713093345457 2024-04-14T11:15:45,650 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#A#compaction#122 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:45,651 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/fdddf02d529640ccb8809635519e3c35 is 50, key is test_row_0/A:col10/1713093345459/Put/seqid=0 2024-04-14T11:15:45,652 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5407acadb5e9c9af78221a8f2d7e6785#B#compaction#123 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:45,653 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/458d764fc7524f60b9f3cd0b15a5dd37 is 50, key is test_row_0/B:col10/1713093345459/Put/seqid=0 2024-04-14T11:15:45,655 DEBUG [Thread-155 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x008f2bdd to 127.0.0.1:60913 2024-04-14T11:15:45,655 DEBUG [Thread-155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:15:45,656 DEBUG [Thread-153 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x036ba6f6 to 127.0.0.1:60913 2024-04-14T11:15:45,656 DEBUG [Thread-153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:15:45,658 DEBUG [Thread-159 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x05800936 to 127.0.0.1:60913 2024-04-14T11:15:45,658 DEBUG [Thread-159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:15:45,660 DEBUG [Thread-157 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1d7aa563 to 127.0.0.1:60913 2024-04-14T11:15:45,660 DEBUG [Thread-157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:15:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=28 2024-04-14T11:15:45,742 INFO [Thread-152 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 28 completed 2024-04-14T11:15:45,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:45,795 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:45,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-04-14T11:15:45,796 DEBUG [Thread-148 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x263d3c32 to 127.0.0.1:60913 2024-04-14T11:15:45,796 DEBUG [Thread-148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:15:45,797 DEBUG [Thread-146 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x72f2ac7d to 127.0.0.1:60913 2024-04-14T11:15:45,797 DEBUG [Thread-146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:15:45,797 DEBUG [Thread-144 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3310b2d1 to 127.0.0.1:60913 2024-04-14T11:15:45,797 DEBUG [Thread-144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:15:45,798 DEBUG [Thread-142 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2d8ad5a3 to 127.0.0.1:60913 2024-04-14T11:15:45,798 DEBUG [Thread-142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:15:45,799 DEBUG [Thread-150 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6b82d154 to 127.0.0.1:60913 2024-04-14T11:15:45,799 DEBUG [Thread-150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:15:45,800 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-04-14T11:15:45,800 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-04-14T11:15:45,800 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-04-14T11:15:45,800 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 126 2024-04-14T11:15:45,800 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-04-14T11:15:45,800 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-04-14T11:15:45,800 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-04-14T11:15:45,800 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(398): read 6141 2024-04-14T11:15:45,800 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(398): read 6109 2024-04-14T11:15:45,800 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-04-14T11:15:45,800 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(402): scanned 2841 2024-04-14T11:15:45,800 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(403): verified 8516 rows 2024-04-14T11:15:45,800 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(402): scanned 2829 2024-04-14T11:15:45,800 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(403): verified 8483 rows 2024-04-14T11:15:45,800 INFO [Listener at localhost/34025 {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-04-14T11:15:45,800 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7cb13939 to 127.0.0.1:60913 2024-04-14T11:15:45,801 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:15:45,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/e6355ddd00234332866a6ea7569276b8 is 50, key is test_row_0/A:col10/1713093345796/Put/seqid=0 2024-04-14T11:15:45,806 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$18(968): Started disable of TestAcidGuarantees 2024-04-14T11:15:45,812 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$13(2744): Client=jenkins//172.31.2.20 disable TestAcidGuarantees 2024-04-14T11:15:45,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=30, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-04-14T11:15:45,824 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093345824"}]},"ts":"1713093345824"} 2024-04-14T11:15:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=30 2024-04-14T11:15:45,825 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-04-14T11:15:45,828 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set TestAcidGuarantees to state=DISABLING 2024-04-14T11:15:45,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=5407acadb5e9c9af78221a8f2d7e6785, UNASSIGN}] 2024-04-14T11:15:45,830 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(729): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=5407acadb5e9c9af78221a8f2d7e6785, UNASSIGN 2024-04-14T11:15:45,831 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=5407acadb5e9c9af78221a8f2d7e6785, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:45,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; CloseRegionProcedure 5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:15:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=30 2024-04-14T11:15:45,989 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:45,992 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(119): Close 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:45,992 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Closing 5407acadb5e9c9af78221a8f2d7e6785, disabling compactions & flushes 2024-04-14T11:15:45,992 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1937): waiting for 2 compactions & cache flush to complete for region TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:46,078 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/fdddf02d529640ccb8809635519e3c35 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/fdddf02d529640ccb8809635519e3c35 2024-04-14T11:15:46,078 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/458d764fc7524f60b9f3cd0b15a5dd37 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/458d764fc7524f60b9f3cd0b15a5dd37 2024-04-14T11:15:46,086 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/B of 5407acadb5e9c9af78221a8f2d7e6785 into 458d764fc7524f60b9f3cd0b15a5dd37(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:46,086 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 5407acadb5e9c9af78221a8f2d7e6785/A of 5407acadb5e9c9af78221a8f2d7e6785 into fdddf02d529640ccb8809635519e3c35(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:46,086 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:46,086 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/B, priority=12, startTime=1713093345620; duration=0sec 2024-04-14T11:15:46,086 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:46,086 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1937): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:46,086 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785., storeName=5407acadb5e9c9af78221a8f2d7e6785/A, priority=12, startTime=1713093345620; duration=0sec 2024-04-14T11:15:46,086 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:46,086 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:B 2024-04-14T11:15:46,086 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. because compaction request was cancelled 2024-04-14T11:15:46,086 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:C 2024-04-14T11:15:46,086 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:46,086 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:A 2024-04-14T11:15:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=30 2024-04-14T11:15:46,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=612 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/e6355ddd00234332866a6ea7569276b8 2024-04-14T11:15:46,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/4f211f305fc044129c1384b4afbef2e1 is 50, key is test_row_0/B:col10/1713093345796/Put/seqid=0 2024-04-14T11:15:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=30 2024-04-14T11:15:46,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=612 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/4f211f305fc044129c1384b4afbef2e1 2024-04-14T11:15:46,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/78f109f5af7944a08bf72593d57a198f is 50, key is test_row_0/C:col10/1713093345796/Put/seqid=0 2024-04-14T11:15:46,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=30 2024-04-14T11:15:47,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=612 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/78f109f5af7944a08bf72593d57a198f 2024-04-14T11:15:47,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/e6355ddd00234332866a6ea7569276b8 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e6355ddd00234332866a6ea7569276b8 2024-04-14T11:15:47,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e6355ddd00234332866a6ea7569276b8, entries=150, sequenceid=612, filesize=12.0 K 2024-04-14T11:15:47,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/4f211f305fc044129c1384b4afbef2e1 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/4f211f305fc044129c1384b4afbef2e1 2024-04-14T11:15:47,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/4f211f305fc044129c1384b4afbef2e1, entries=150, sequenceid=612, filesize=12.0 K 2024-04-14T11:15:47,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/78f109f5af7944a08bf72593d57a198f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/78f109f5af7944a08bf72593d57a198f 2024-04-14T11:15:47,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/78f109f5af7944a08bf72593d57a198f, entries=150, sequenceid=612, filesize=12.0 K 2024-04-14T11:15:47,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=6.71 KB/6870 for 5407acadb5e9c9af78221a8f2d7e6785 in 1278ms, sequenceid=612, compaction requested=true 2024-04-14T11:15:47,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:47,074 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1698): Closing region TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:47,074 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1719): Waiting without time limit for close lock on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:47,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:15:47,074 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1786): Acquired close lock on TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. after waiting 0 ms 2024-04-14T11:15:47,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:47,074 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. because compaction request was cancelled 2024-04-14T11:15:47,074 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1796): Updates disabled for region TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:47,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:15:47,074 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:A 2024-04-14T11:15:47,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:47,074 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. because compaction request was cancelled 2024-04-14T11:15:47,074 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2832): Flushing 5407acadb5e9c9af78221a8f2d7e6785 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-04-14T11:15:47,074 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:B 2024-04-14T11:15:47,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5407acadb5e9c9af78221a8f2d7e6785:C, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:15:47,074 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. because compaction request was cancelled 2024-04-14T11:15:47,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:47,074 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5407acadb5e9c9af78221a8f2d7e6785:C 2024-04-14T11:15:47,078 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/f289554c93de4d799c2c729acee52fdf is 50, key is test_row_1/A:col10/1713093345798/Put/seqid=0 2024-04-14T11:15:47,487 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/f289554c93de4d799c2c729acee52fdf 2024-04-14T11:15:47,496 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/5874cf18a1e14d10b4608769bff34419 is 50, key is test_row_1/B:col10/1713093345798/Put/seqid=0 2024-04-14T11:15:47,901 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/5874cf18a1e14d10b4608769bff34419 2024-04-14T11:15:47,912 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/d4f9f15cc0524f6c89ad35ba935f768b is 50, key is test_row_1/C:col10/1713093345798/Put/seqid=0 2024-04-14T11:15:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=30 2024-04-14T11:15:48,318 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/d4f9f15cc0524f6c89ad35ba935f768b 2024-04-14T11:15:48,325 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/A/f289554c93de4d799c2c729acee52fdf as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/f289554c93de4d799c2c729acee52fdf 2024-04-14T11:15:48,330 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/f289554c93de4d799c2c729acee52fdf, entries=50, sequenceid=618, filesize=7.2 K 2024-04-14T11:15:48,331 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/B/5874cf18a1e14d10b4608769bff34419 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/5874cf18a1e14d10b4608769bff34419 2024-04-14T11:15:48,336 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/5874cf18a1e14d10b4608769bff34419, entries=50, sequenceid=618, filesize=7.2 K 2024-04-14T11:15:48,337 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/.tmp/C/d4f9f15cc0524f6c89ad35ba935f768b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d4f9f15cc0524f6c89ad35ba935f768b 2024-04-14T11:15:48,343 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d4f9f15cc0524f6c89ad35ba935f768b, entries=50, sequenceid=618, filesize=7.2 K 2024-04-14T11:15:48,344 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3035): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 5407acadb5e9c9af78221a8f2d7e6785 in 1270ms, sequenceid=618, compaction requested=true 2024-04-14T11:15:48,360 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d3ef8418b61b4c70bb0bd681a9533a52, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d6d43aeebc7a4f6193bd0e29e133c96d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/9d684e784c0a477b950805be221f3d67, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/ec5c84e30cc949c298749ae2ff1e7388, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/fca166abc2584f37b60e30b8a030cf0f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/2471ce883f8845beae4910af3241582f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/c594b3fcf2d242fe88ecd4277a2abf70, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/eaac03084b5240b788c680caeebf0929, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/571194c6d49d4c87be9f2b65374acf59, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/23410428c3204b3a912a55e693429a25, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/da3ffb172b8441ba81296a3bd44105cd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/815ca5f592ef4872b34b0685988d2e52, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/7faf64ea0c854a7aa777bd63bcc61279, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/71660db0547d452e9043f211ede5c674, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/50d4d8d7b8614e4ea7babc6ca6bf0069, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cecaf62a39974a2ea37fbaaf4ea9e884, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/55784b1558dd46f18ec089dba5bf229d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cf92cb3750ac45e5862b50a733ffc624, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/5e95cac412e040e0b7eb8478f849b603, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e372b9d0582b4a6c97dc34d7758b6243, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e1a80d1d0d3a42d1bda76494dcf07baf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d8f6bc1a89684f939e31c7e2de9ccdb3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/b882ad39904f48b3bb8ae6f6dcd145d8, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/6f367ce6b47a48f4819ea24a08758ab8, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/05894ea305e341cca653cdf56d41d1c5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/93aa130b8586483d809402ac86519ef1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cac5e5910e614066a0ce92419ce131b4, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/84082eec732346fcab80a00a9c0fd4fc, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/6c66551efc454be39b9d6bf06019cdf4, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d9d1282edf374fa7a225d6da86eb07cf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/4c941a687be14eeabb526c6a40386578, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/2b5c1c25d4f748689299eb5815b51dd7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/725b338633a54c0484a33b1619d43adc, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cf2f22a645ab4a4196134ddc3e753ed5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cb8c0c7bea83417da9774e93c860bcd0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e3c3d26fc32445bdae7dacd7e6500c61, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/8c5de8a7b8be46cca33e270e79954d3b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/449550338a6d4d45ae074badb604473e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/264948e66a7c419bbf4c0df231c4ac25, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/eb3661eb132d46daafd0b8c955a7b25d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/3162da97f5c843c0ac163effbb012aba] to archive 2024-04-14T11:15:48,364 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:15:48,372 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d3ef8418b61b4c70bb0bd681a9533a52 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d3ef8418b61b4c70bb0bd681a9533a52 2024-04-14T11:15:48,374 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d6d43aeebc7a4f6193bd0e29e133c96d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d6d43aeebc7a4f6193bd0e29e133c96d 2024-04-14T11:15:48,375 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/9d684e784c0a477b950805be221f3d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/9d684e784c0a477b950805be221f3d67 2024-04-14T11:15:48,377 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/ec5c84e30cc949c298749ae2ff1e7388 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/ec5c84e30cc949c298749ae2ff1e7388 2024-04-14T11:15:48,379 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/fca166abc2584f37b60e30b8a030cf0f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/fca166abc2584f37b60e30b8a030cf0f 2024-04-14T11:15:48,381 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/2471ce883f8845beae4910af3241582f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/2471ce883f8845beae4910af3241582f 2024-04-14T11:15:48,383 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/c594b3fcf2d242fe88ecd4277a2abf70 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/c594b3fcf2d242fe88ecd4277a2abf70 2024-04-14T11:15:48,384 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/eaac03084b5240b788c680caeebf0929 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/eaac03084b5240b788c680caeebf0929 2024-04-14T11:15:48,386 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/571194c6d49d4c87be9f2b65374acf59 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/571194c6d49d4c87be9f2b65374acf59 2024-04-14T11:15:48,387 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/23410428c3204b3a912a55e693429a25 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/23410428c3204b3a912a55e693429a25 2024-04-14T11:15:48,389 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/da3ffb172b8441ba81296a3bd44105cd to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/da3ffb172b8441ba81296a3bd44105cd 2024-04-14T11:15:48,390 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/815ca5f592ef4872b34b0685988d2e52 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/815ca5f592ef4872b34b0685988d2e52 2024-04-14T11:15:48,392 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/7faf64ea0c854a7aa777bd63bcc61279 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/7faf64ea0c854a7aa777bd63bcc61279 2024-04-14T11:15:48,393 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/71660db0547d452e9043f211ede5c674 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/71660db0547d452e9043f211ede5c674 2024-04-14T11:15:48,395 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/50d4d8d7b8614e4ea7babc6ca6bf0069 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/50d4d8d7b8614e4ea7babc6ca6bf0069 2024-04-14T11:15:48,397 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cecaf62a39974a2ea37fbaaf4ea9e884 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cecaf62a39974a2ea37fbaaf4ea9e884 2024-04-14T11:15:48,398 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/55784b1558dd46f18ec089dba5bf229d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/55784b1558dd46f18ec089dba5bf229d 2024-04-14T11:15:48,400 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cf92cb3750ac45e5862b50a733ffc624 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cf92cb3750ac45e5862b50a733ffc624 2024-04-14T11:15:48,401 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/5e95cac412e040e0b7eb8478f849b603 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/5e95cac412e040e0b7eb8478f849b603 2024-04-14T11:15:48,402 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e372b9d0582b4a6c97dc34d7758b6243 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e372b9d0582b4a6c97dc34d7758b6243 2024-04-14T11:15:48,404 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e1a80d1d0d3a42d1bda76494dcf07baf to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e1a80d1d0d3a42d1bda76494dcf07baf 2024-04-14T11:15:48,405 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d8f6bc1a89684f939e31c7e2de9ccdb3 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d8f6bc1a89684f939e31c7e2de9ccdb3 2024-04-14T11:15:48,407 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/b882ad39904f48b3bb8ae6f6dcd145d8 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/b882ad39904f48b3bb8ae6f6dcd145d8 2024-04-14T11:15:48,408 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/6f367ce6b47a48f4819ea24a08758ab8 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/6f367ce6b47a48f4819ea24a08758ab8 2024-04-14T11:15:48,410 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/05894ea305e341cca653cdf56d41d1c5 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/05894ea305e341cca653cdf56d41d1c5 2024-04-14T11:15:48,411 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/93aa130b8586483d809402ac86519ef1 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/93aa130b8586483d809402ac86519ef1 2024-04-14T11:15:48,412 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cac5e5910e614066a0ce92419ce131b4 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cac5e5910e614066a0ce92419ce131b4 2024-04-14T11:15:48,414 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/84082eec732346fcab80a00a9c0fd4fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/84082eec732346fcab80a00a9c0fd4fc 2024-04-14T11:15:48,415 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/6c66551efc454be39b9d6bf06019cdf4 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/6c66551efc454be39b9d6bf06019cdf4 2024-04-14T11:15:48,416 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d9d1282edf374fa7a225d6da86eb07cf to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/d9d1282edf374fa7a225d6da86eb07cf 2024-04-14T11:15:48,418 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/4c941a687be14eeabb526c6a40386578 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/4c941a687be14eeabb526c6a40386578 2024-04-14T11:15:48,419 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/2b5c1c25d4f748689299eb5815b51dd7 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/2b5c1c25d4f748689299eb5815b51dd7 2024-04-14T11:15:48,420 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/725b338633a54c0484a33b1619d43adc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/725b338633a54c0484a33b1619d43adc 2024-04-14T11:15:48,422 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cf2f22a645ab4a4196134ddc3e753ed5 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cf2f22a645ab4a4196134ddc3e753ed5 2024-04-14T11:15:48,423 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cb8c0c7bea83417da9774e93c860bcd0 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/cb8c0c7bea83417da9774e93c860bcd0 2024-04-14T11:15:48,424 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e3c3d26fc32445bdae7dacd7e6500c61 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e3c3d26fc32445bdae7dacd7e6500c61 2024-04-14T11:15:48,426 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/8c5de8a7b8be46cca33e270e79954d3b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/8c5de8a7b8be46cca33e270e79954d3b 2024-04-14T11:15:48,427 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/449550338a6d4d45ae074badb604473e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/449550338a6d4d45ae074badb604473e 2024-04-14T11:15:48,428 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/264948e66a7c419bbf4c0df231c4ac25 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/264948e66a7c419bbf4c0df231c4ac25 2024-04-14T11:15:48,430 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/eb3661eb132d46daafd0b8c955a7b25d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/eb3661eb132d46daafd0b8c955a7b25d 2024-04-14T11:15:48,431 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/3162da97f5c843c0ac163effbb012aba to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/3162da97f5c843c0ac163effbb012aba 2024-04-14T11:15:48,454 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/e98a4f4f27f94c429bc7c23966ad7630, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/33a2fc35a4d041fe9ef7799aba56cd44, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/d7d55f3f334a4a6fb55ef754f2f5cb91, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/106ad7c3b715403681b187833e591a3b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/6a684de6aad542b0a1cb4b84b19889a1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c338873574c34a5dbf153b6ba866287d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c15b907cd8284e0da65826d5a539747e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/799fdcf2fc07401880bae6e97adc0f5e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/f50ea39f038146d3a3db6b0d14b7c946, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/967246fad01f47af96f4b01288dc7c76, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/82ce418fe4ec495d998036f8a6852395, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/6beec88b1aa34e1f84db1955b37fe958, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c79e28636b8848ceb7bd8bdcb7b17eab, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/6181033228884261a35a4b48e6142e85, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/3b9645903ef348c3a0cba98ae5233113, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/9592fbea2d6c48fa9fcc27b8a9bdfeb5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/ef5271ae8d9c4c2cb2923aab89027bdf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/078dfeee46304bd981d3bfa15b77e68a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/be17b3be30a34446abc1547814716abc, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/58e7ab9d98734338bdd1979179878189, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/06abaa706f8d49fd96c49455319c5adf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c92d4011bfd54413be80fe3e2303b473, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/b5694b97dbe14280974f6102586b03d0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/af22681d8e844e83a572d5dfce7e79f4, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/a8e70c305c43457d95bcc73f1bcd8054, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/094d1c403cb2473da01efda5de7d62ae, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/00603b192bdf4b81be57157af214dc19, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/a9c57e0a63a449d0b8961f441f01d935, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/bff7b47a39884902907941b22e552258, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/9ba07f2bb32f4e05910f9708cd23501f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/173a10f2885a4787bb9f2f2b849bf6ba, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/64106d0e214b4044bccf04afbfdfd6dc, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/b01dbfd5c2bb487783ec501e0a0fa30c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/f363bb1e37c54a2ca472e9e64853409e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/3fc6d7387d5247b598dea7c36382127d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/aff142545c904468a3d9f10a5320b707, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/83f5d9251b6b43e5889399e33e5bede3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/ba7f1b8206114905b23eb506c3e32b99, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/ffc38569b95446a8af30f67e0d864e34, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/e635248d188a4cab98f972f10938c6c1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/3dce0c80eb664c5db501c936c875bdb9] to archive 2024-04-14T11:15:48,456 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:15:48,458 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/e98a4f4f27f94c429bc7c23966ad7630 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/e98a4f4f27f94c429bc7c23966ad7630 2024-04-14T11:15:48,460 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/33a2fc35a4d041fe9ef7799aba56cd44 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/33a2fc35a4d041fe9ef7799aba56cd44 2024-04-14T11:15:48,461 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/d7d55f3f334a4a6fb55ef754f2f5cb91 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/d7d55f3f334a4a6fb55ef754f2f5cb91 2024-04-14T11:15:48,463 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/106ad7c3b715403681b187833e591a3b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/106ad7c3b715403681b187833e591a3b 2024-04-14T11:15:48,464 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/6a684de6aad542b0a1cb4b84b19889a1 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/6a684de6aad542b0a1cb4b84b19889a1 2024-04-14T11:15:48,466 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c338873574c34a5dbf153b6ba866287d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c338873574c34a5dbf153b6ba866287d 2024-04-14T11:15:48,467 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c15b907cd8284e0da65826d5a539747e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c15b907cd8284e0da65826d5a539747e 2024-04-14T11:15:48,469 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/799fdcf2fc07401880bae6e97adc0f5e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/799fdcf2fc07401880bae6e97adc0f5e 2024-04-14T11:15:48,471 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/f50ea39f038146d3a3db6b0d14b7c946 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/f50ea39f038146d3a3db6b0d14b7c946 2024-04-14T11:15:48,472 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/967246fad01f47af96f4b01288dc7c76 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/967246fad01f47af96f4b01288dc7c76 2024-04-14T11:15:48,474 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/82ce418fe4ec495d998036f8a6852395 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/82ce418fe4ec495d998036f8a6852395 2024-04-14T11:15:48,475 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/6beec88b1aa34e1f84db1955b37fe958 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/6beec88b1aa34e1f84db1955b37fe958 2024-04-14T11:15:48,477 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c79e28636b8848ceb7bd8bdcb7b17eab to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c79e28636b8848ceb7bd8bdcb7b17eab 2024-04-14T11:15:48,478 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/6181033228884261a35a4b48e6142e85 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/6181033228884261a35a4b48e6142e85 2024-04-14T11:15:48,480 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/3b9645903ef348c3a0cba98ae5233113 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/3b9645903ef348c3a0cba98ae5233113 2024-04-14T11:15:48,481 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/9592fbea2d6c48fa9fcc27b8a9bdfeb5 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/9592fbea2d6c48fa9fcc27b8a9bdfeb5 2024-04-14T11:15:48,483 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/ef5271ae8d9c4c2cb2923aab89027bdf to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/ef5271ae8d9c4c2cb2923aab89027bdf 2024-04-14T11:15:48,484 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/078dfeee46304bd981d3bfa15b77e68a to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/078dfeee46304bd981d3bfa15b77e68a 2024-04-14T11:15:48,486 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/be17b3be30a34446abc1547814716abc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/be17b3be30a34446abc1547814716abc 2024-04-14T11:15:48,488 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/58e7ab9d98734338bdd1979179878189 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/58e7ab9d98734338bdd1979179878189 2024-04-14T11:15:48,489 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/06abaa706f8d49fd96c49455319c5adf to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/06abaa706f8d49fd96c49455319c5adf 2024-04-14T11:15:48,491 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c92d4011bfd54413be80fe3e2303b473 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/c92d4011bfd54413be80fe3e2303b473 2024-04-14T11:15:48,492 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/b5694b97dbe14280974f6102586b03d0 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/b5694b97dbe14280974f6102586b03d0 2024-04-14T11:15:48,494 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/af22681d8e844e83a572d5dfce7e79f4 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/af22681d8e844e83a572d5dfce7e79f4 2024-04-14T11:15:48,495 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/a8e70c305c43457d95bcc73f1bcd8054 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/a8e70c305c43457d95bcc73f1bcd8054 2024-04-14T11:15:48,497 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/094d1c403cb2473da01efda5de7d62ae to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/094d1c403cb2473da01efda5de7d62ae 2024-04-14T11:15:48,498 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/00603b192bdf4b81be57157af214dc19 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/00603b192bdf4b81be57157af214dc19 2024-04-14T11:15:48,500 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/a9c57e0a63a449d0b8961f441f01d935 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/a9c57e0a63a449d0b8961f441f01d935 2024-04-14T11:15:48,501 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/bff7b47a39884902907941b22e552258 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/bff7b47a39884902907941b22e552258 2024-04-14T11:15:48,502 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/9ba07f2bb32f4e05910f9708cd23501f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/9ba07f2bb32f4e05910f9708cd23501f 2024-04-14T11:15:48,504 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/173a10f2885a4787bb9f2f2b849bf6ba to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/173a10f2885a4787bb9f2f2b849bf6ba 2024-04-14T11:15:48,505 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/64106d0e214b4044bccf04afbfdfd6dc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/64106d0e214b4044bccf04afbfdfd6dc 2024-04-14T11:15:48,506 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/b01dbfd5c2bb487783ec501e0a0fa30c to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/b01dbfd5c2bb487783ec501e0a0fa30c 2024-04-14T11:15:48,508 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/f363bb1e37c54a2ca472e9e64853409e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/f363bb1e37c54a2ca472e9e64853409e 2024-04-14T11:15:48,509 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/3fc6d7387d5247b598dea7c36382127d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/3fc6d7387d5247b598dea7c36382127d 2024-04-14T11:15:48,510 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/aff142545c904468a3d9f10a5320b707 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/aff142545c904468a3d9f10a5320b707 2024-04-14T11:15:48,512 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/83f5d9251b6b43e5889399e33e5bede3 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/83f5d9251b6b43e5889399e33e5bede3 2024-04-14T11:15:48,513 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/ba7f1b8206114905b23eb506c3e32b99 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/ba7f1b8206114905b23eb506c3e32b99 2024-04-14T11:15:48,514 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/ffc38569b95446a8af30f67e0d864e34 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/ffc38569b95446a8af30f67e0d864e34 2024-04-14T11:15:48,515 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/e635248d188a4cab98f972f10938c6c1 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/e635248d188a4cab98f972f10938c6c1 2024-04-14T11:15:48,517 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/3dce0c80eb664c5db501c936c875bdb9 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/3dce0c80eb664c5db501c936c875bdb9 2024-04-14T11:15:48,519 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/ba3418be93184e94931f2e43daad8b12, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/9a5da16d52b74a6a9868518794e59e82, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/116f47cf8d894c6aa7a8cecadd4fef48, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/f8952897c30b47ebb31cd6815503c07c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/e7b7edc5c03e4d59b14e5e000b0701e2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/953d1ddb0c644b88befffa1b900e6ecf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/639031aa312b4b33a22d855a8eba60bd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/baf8a0a169d44afb80c552eb1b0940dd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d24b6633330543989f4de98465c80166, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/4e230dad1d0b481189dc9fd5f9284630, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/2237c363f8c1425c8d301bdb40368661, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/882e5627eb904d1eade4afbd61bfdfec, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/bfea52c14f784dedbb596470764f4516, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/9f6c79a221a144a0a7e5ef01f1af215f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/5159079a347740da8ff362dc143c5643, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/41a8b3ea7f7f43aeb057d7340643a06b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/1e08ac2e49d24d2a880faf3191a1734c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/c57ee0796325455f9cbf0f789243ef3e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d9864ac9d5db4600972d8564e4d0b7cc, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/5dfa14b0c9994e87a34e9d374d017087, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/8d94324f81244e4291ae886defcc0fc0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d2bb722d630444b2a8cd7bcbad842170, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/909ff096419d48bdb07670703fb81c07, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/511790b2278b4c498cdb968ff9331e2b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/e4efefd6149d4482a5e8327723df9c3f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/03f9128c67334e42b176beb1f4a9ae44, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/c767ef9c1a2c4827a8ab3ef41107d9ed, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/facbe1c980d44ab0b364a78c9154f256, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/e92d7a79ab474ce7aabd985ac4a26dee, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/be1c4ddc3b6243199128c3c8f048c155, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/bd66300bc2d04153be7e7a600064813a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/1ce724858c6f4ec9a8f0e3b62be37032, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/2b04186d45c34ca3b2b96f54e298bd38, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/46202302ce4a46f0bdfad9a67519caf7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/f180c9cf91364798907b0b42dcb301e8, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/b8b09bd80daa45618b899026d933ca7f] to archive 2024-04-14T11:15:48,520 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:15:48,522 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/ba3418be93184e94931f2e43daad8b12 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/ba3418be93184e94931f2e43daad8b12 2024-04-14T11:15:48,523 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/9a5da16d52b74a6a9868518794e59e82 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/9a5da16d52b74a6a9868518794e59e82 2024-04-14T11:15:48,524 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/116f47cf8d894c6aa7a8cecadd4fef48 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/116f47cf8d894c6aa7a8cecadd4fef48 2024-04-14T11:15:48,526 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/f8952897c30b47ebb31cd6815503c07c to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/f8952897c30b47ebb31cd6815503c07c 2024-04-14T11:15:48,527 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/e7b7edc5c03e4d59b14e5e000b0701e2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/e7b7edc5c03e4d59b14e5e000b0701e2 2024-04-14T11:15:48,528 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/953d1ddb0c644b88befffa1b900e6ecf to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/953d1ddb0c644b88befffa1b900e6ecf 2024-04-14T11:15:48,530 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/639031aa312b4b33a22d855a8eba60bd to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/639031aa312b4b33a22d855a8eba60bd 2024-04-14T11:15:48,531 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/baf8a0a169d44afb80c552eb1b0940dd to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/baf8a0a169d44afb80c552eb1b0940dd 2024-04-14T11:15:48,533 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d24b6633330543989f4de98465c80166 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d24b6633330543989f4de98465c80166 2024-04-14T11:15:48,534 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/4e230dad1d0b481189dc9fd5f9284630 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/4e230dad1d0b481189dc9fd5f9284630 2024-04-14T11:15:48,536 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/2237c363f8c1425c8d301bdb40368661 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/2237c363f8c1425c8d301bdb40368661 2024-04-14T11:15:48,537 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/882e5627eb904d1eade4afbd61bfdfec to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/882e5627eb904d1eade4afbd61bfdfec 2024-04-14T11:15:48,539 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/bfea52c14f784dedbb596470764f4516 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/bfea52c14f784dedbb596470764f4516 2024-04-14T11:15:48,540 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/9f6c79a221a144a0a7e5ef01f1af215f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/9f6c79a221a144a0a7e5ef01f1af215f 2024-04-14T11:15:48,541 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/5159079a347740da8ff362dc143c5643 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/5159079a347740da8ff362dc143c5643 2024-04-14T11:15:48,543 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/41a8b3ea7f7f43aeb057d7340643a06b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/41a8b3ea7f7f43aeb057d7340643a06b 2024-04-14T11:15:48,544 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/1e08ac2e49d24d2a880faf3191a1734c to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/1e08ac2e49d24d2a880faf3191a1734c 2024-04-14T11:15:48,546 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/c57ee0796325455f9cbf0f789243ef3e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/c57ee0796325455f9cbf0f789243ef3e 2024-04-14T11:15:48,547 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d9864ac9d5db4600972d8564e4d0b7cc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d9864ac9d5db4600972d8564e4d0b7cc 2024-04-14T11:15:48,548 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/5dfa14b0c9994e87a34e9d374d017087 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/5dfa14b0c9994e87a34e9d374d017087 2024-04-14T11:15:48,550 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/8d94324f81244e4291ae886defcc0fc0 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/8d94324f81244e4291ae886defcc0fc0 2024-04-14T11:15:48,551 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d2bb722d630444b2a8cd7bcbad842170 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d2bb722d630444b2a8cd7bcbad842170 2024-04-14T11:15:48,553 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/909ff096419d48bdb07670703fb81c07 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/909ff096419d48bdb07670703fb81c07 2024-04-14T11:15:48,554 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/511790b2278b4c498cdb968ff9331e2b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/511790b2278b4c498cdb968ff9331e2b 2024-04-14T11:15:48,556 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/e4efefd6149d4482a5e8327723df9c3f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/e4efefd6149d4482a5e8327723df9c3f 2024-04-14T11:15:48,557 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/03f9128c67334e42b176beb1f4a9ae44 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/03f9128c67334e42b176beb1f4a9ae44 2024-04-14T11:15:48,559 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/c767ef9c1a2c4827a8ab3ef41107d9ed to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/c767ef9c1a2c4827a8ab3ef41107d9ed 2024-04-14T11:15:48,560 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/facbe1c980d44ab0b364a78c9154f256 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/facbe1c980d44ab0b364a78c9154f256 2024-04-14T11:15:48,562 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/e92d7a79ab474ce7aabd985ac4a26dee to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/e92d7a79ab474ce7aabd985ac4a26dee 2024-04-14T11:15:48,563 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/be1c4ddc3b6243199128c3c8f048c155 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/be1c4ddc3b6243199128c3c8f048c155 2024-04-14T11:15:48,565 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/bd66300bc2d04153be7e7a600064813a to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/bd66300bc2d04153be7e7a600064813a 2024-04-14T11:15:48,566 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/1ce724858c6f4ec9a8f0e3b62be37032 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/1ce724858c6f4ec9a8f0e3b62be37032 2024-04-14T11:15:48,568 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/2b04186d45c34ca3b2b96f54e298bd38 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/2b04186d45c34ca3b2b96f54e298bd38 2024-04-14T11:15:48,569 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/46202302ce4a46f0bdfad9a67519caf7 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/46202302ce4a46f0bdfad9a67519caf7 2024-04-14T11:15:48,571 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/f180c9cf91364798907b0b42dcb301e8 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/f180c9cf91364798907b0b42dcb301e8 2024-04-14T11:15:48,572 DEBUG [StoreCloser-TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/b8b09bd80daa45618b899026d933ca7f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/b8b09bd80daa45618b899026d933ca7f 2024-04-14T11:15:48,577 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/recovered.edits/621.seqid, newMaxSeqId=621, maxSeqId=1 2024-04-14T11:15:48,579 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1917): Closed TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785. 2024-04-14T11:15:48,579 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1630): Region close journal for 5407acadb5e9c9af78221a8f2d7e6785: 2024-04-14T11:15:48,582 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(163): Closed 5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:48,582 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=31 updating hbase:meta row=5407acadb5e9c9af78221a8f2d7e6785, regionState=CLOSED 2024-04-14T11:15:48,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=32, resume processing ppid=31 2024-04-14T11:15:48,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=32, ppid=31, state=SUCCESS; CloseRegionProcedure 5407acadb5e9c9af78221a8f2d7e6785, server=jenkins-hbase6.apache.org,33239,1713093321027 in 2.7510 sec 2024-04-14T11:15:48,587 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=31, resume processing ppid=30 2024-04-14T11:15:48,587 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=31, ppid=30, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=5407acadb5e9c9af78221a8f2d7e6785, UNASSIGN in 2.7560 sec 2024-04-14T11:15:48,588 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093348587"}]},"ts":"1713093348587"} 2024-04-14T11:15:48,589 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-04-14T11:15:48,591 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set TestAcidGuarantees to state=DISABLED 2024-04-14T11:15:48,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=30, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 2.7780 sec 2024-04-14T11:15:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=30 2024-04-14T11:15:49,933 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$TableFuture(3746): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 30 completed 2024-04-14T11:15:49,938 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$5(2494): Client=jenkins//172.31.2.20 delete TestAcidGuarantees 2024-04-14T11:15:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:15:49,947 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:15:49,949 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:15:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=33 2024-04-14T11:15:49,953 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:49,959 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A, FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B, FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C, FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/recovered.edits] 2024-04-14T11:15:49,964 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e6355ddd00234332866a6ea7569276b8 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/e6355ddd00234332866a6ea7569276b8 2024-04-14T11:15:49,966 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/f289554c93de4d799c2c729acee52fdf to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/f289554c93de4d799c2c729acee52fdf 2024-04-14T11:15:49,968 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/fdddf02d529640ccb8809635519e3c35 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/A/fdddf02d529640ccb8809635519e3c35 2024-04-14T11:15:49,971 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/458d764fc7524f60b9f3cd0b15a5dd37 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/458d764fc7524f60b9f3cd0b15a5dd37 2024-04-14T11:15:49,973 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/4f211f305fc044129c1384b4afbef2e1 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/4f211f305fc044129c1384b4afbef2e1 2024-04-14T11:15:49,975 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/5874cf18a1e14d10b4608769bff34419 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/B/5874cf18a1e14d10b4608769bff34419 2024-04-14T11:15:49,979 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/37a901cad1ac415baa53c194537a9e9c to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/37a901cad1ac415baa53c194537a9e9c 2024-04-14T11:15:49,981 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/747af3e3244b48b5a9847da90b59fd78 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/747af3e3244b48b5a9847da90b59fd78 2024-04-14T11:15:49,983 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/78f109f5af7944a08bf72593d57a198f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/78f109f5af7944a08bf72593d57a198f 2024-04-14T11:15:49,985 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/7aab326791d6413cb3c0b004084936c4 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/7aab326791d6413cb3c0b004084936c4 2024-04-14T11:15:49,987 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/90a550f867004f019721b35ca7841746 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/90a550f867004f019721b35ca7841746 2024-04-14T11:15:49,988 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d4f9f15cc0524f6c89ad35ba935f768b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/C/d4f9f15cc0524f6c89ad35ba935f768b 2024-04-14T11:15:49,992 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/recovered.edits/621.seqid to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785/recovered.edits/621.seqid 2024-04-14T11:15:49,993 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/5407acadb5e9c9af78221a8f2d7e6785 2024-04-14T11:15:49,993 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived TestAcidGuarantees regions 2024-04-14T11:15:50,002 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:15:50,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-04-14T11:15:50,013 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-04-14T11:15:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=33 2024-04-14T11:15:50,066 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'TestAcidGuarantees' descriptor. 2024-04-14T11:15:50,068 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:15:50,068 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'TestAcidGuarantees' from region states. 2024-04-14T11:15:50,068 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1713093350068"}]},"ts":"9223372036854775807"} 2024-04-14T11:15:50,071 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-04-14T11:15:50,072 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 5407acadb5e9c9af78221a8f2d7e6785, NAME => 'TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785.', STARTKEY => '', ENDKEY => ''}] 2024-04-14T11:15:50,072 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'TestAcidGuarantees' as deleted. 2024-04-14T11:15:50,072 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1713093350072"}]},"ts":"9223372036854775807"} 2024-04-14T11:15:50,075 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-04-14T11:15:50,078 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(127): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:15:50,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=33, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 139 msec 2024-04-14T11:15:50,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=33 2024-04-14T11:15:50,253 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$TableFuture(3746): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 33 completed 2024-04-14T11:15:50,275 INFO [Listener at localhost/34025 {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=223 (was 205) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_489569842_18 at /127.0.0.1:43958 [Waiting for operation #302] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS-EventLoopGroup-1-3 org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:209) org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:202) org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:316) org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:373) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS:0;jenkins-hbase6:33239-shortCompactions-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:549) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_489569842_18 at /127.0.0.1:42312 [Waiting for operation #238] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7a2db419-shared-pool-4 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1232019390_18 at /127.0.0.1:42170 [Waiting for operation #286] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RSProcedureDispatcher-pool-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7a2db419-shared-pool-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer for 'HBase' metrics system java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7a2db419-shared-pool-7 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7a2db419-shared-pool-5 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=525 (was 506) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=381 (was 316) - SystemLoadAverage LEAK? -, ProcessCount=168 (was 168), AvailableMemoryMB=2438 (was 2885) 2024-04-14T11:15:50,290 INFO [Listener at localhost/34025 {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=223, OpenFileDescriptor=525, MaxFileDescriptor=60000, SystemLoadAverage=381, ProcessCount=168, AvailableMemoryMB=2438 2024-04-14T11:15:50,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-04-14T11:15:50,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$4(2378): Client=jenkins//172.31.2.20 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-04-14T11:15:50,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-04-14T11:15:50,297 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(84): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-04-14T11:15:50,297 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:50,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-04-14T11:15:50,298 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(84): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-04-14T11:15:50,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2024-04-14T11:15:50,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2024-04-14T11:15:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2024-04-14T11:15:50,707 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-04-14T11:15:50,709 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7088): creating {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a 2024-04-14T11:15:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2024-04-14T11:15:51,117 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(891): Instantiated TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:15:51,117 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Closing 4dbfc9ae90980452da73dfe0910874fc, disabling compactions & flushes 2024-04-14T11:15:51,117 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1698): Closing region TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:51,117 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1719): Waiting without time limit for close lock on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:51,117 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1786): Acquired close lock on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. after waiting 0 ms 2024-04-14T11:15:51,117 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1796): Updates disabled for region TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:51,117 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1917): Closed TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:51,117 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1630): Region close journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:15:51,119 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(84): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-04-14T11:15:51,119 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1713093351119"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1713093351119"}]},"ts":"1713093351119"} 2024-04-14T11:15:51,121 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-04-14T11:15:51,121 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(84): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-04-14T11:15:51,122 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093351121"}]},"ts":"1713093351121"} 2024-04-14T11:15:51,123 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-04-14T11:15:51,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=4dbfc9ae90980452da73dfe0910874fc, ASSIGN}] 2024-04-14T11:15:51,131 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(729): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=4dbfc9ae90980452da73dfe0910874fc, ASSIGN 2024-04-14T11:15:51,132 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4dbfc9ae90980452da73dfe0910874fc, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,33239,1713093321027; forceNewPlan=false, retain=false 2024-04-14T11:15:51,283 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=4dbfc9ae90980452da73dfe0910874fc, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:51,285 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE; OpenRegionProcedure 4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:15:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2024-04-14T11:15:51,437 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:51,441 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:51,441 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7267): Opening region: {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} 2024-04-14T11:15:51,442 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:51,442 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(891): Instantiated TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:15:51,442 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7309): checking encryption for 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:51,442 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7312): checking classloading for 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:51,443 INFO [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:51,445 INFO [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dbfc9ae90980452da73dfe0910874fc columnFamilyName A 2024-04-14T11:15:51,445 DEBUG [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:51,446 INFO [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] regionserver.HStore(324): Store=4dbfc9ae90980452da73dfe0910874fc/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:15:51,446 INFO [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:51,448 INFO [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dbfc9ae90980452da73dfe0910874fc columnFamilyName B 2024-04-14T11:15:51,448 DEBUG [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:51,449 INFO [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] regionserver.HStore(324): Store=4dbfc9ae90980452da73dfe0910874fc/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:15:51,449 INFO [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:51,450 INFO [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dbfc9ae90980452da73dfe0910874fc columnFamilyName C 2024-04-14T11:15:51,451 DEBUG [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:51,451 INFO [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] regionserver.HStore(324): Store=4dbfc9ae90980452da73dfe0910874fc/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:15:51,452 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:51,452 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:51,454 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-04-14T11:15:51,456 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1082): writing seq id for 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:51,458 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-04-14T11:15:51,458 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1099): Opened 4dbfc9ae90980452da73dfe0910874fc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71365549, jitterRate=0.06342954933643341}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-04-14T11:15:51,459 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(998): Region open journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:15:51,460 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2593): Post open deploy tasks for TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., pid=36, masterSystemTime=1713093351437 2024-04-14T11:15:51,461 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2620): Finished post open deploy task for TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:51,462 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:51,462 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=4dbfc9ae90980452da73dfe0910874fc, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:51,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=36, resume processing ppid=35 2024-04-14T11:15:51,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=36, ppid=35, state=SUCCESS; OpenRegionProcedure 4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 in 178 msec 2024-04-14T11:15:51,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=35, resume processing ppid=34 2024-04-14T11:15:51,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=35, ppid=34, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=4dbfc9ae90980452da73dfe0910874fc, ASSIGN in 335 msec 2024-04-14T11:15:51,468 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(84): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-04-14T11:15:51,468 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093351468"}]},"ts":"1713093351468"} 2024-04-14T11:15:51,470 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-04-14T11:15:51,473 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(84): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-04-14T11:15:51,474 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=34, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-04-14T11:15:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2024-04-14T11:15:52,406 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 34 completed 2024-04-14T11:15:52,408 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x752ce290 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:52,423 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57d9bb26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:52,427 DEBUG [Listener at localhost/34025 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:52,432 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:37320, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:52,434 DEBUG [Listener at localhost/34025 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-04-14T11:15:52,436 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:37568, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-04-14T11:15:52,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-04-14T11:15:52,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$14(2787): Client=jenkins//172.31.2.20 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-04-14T11:15:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=TestAcidGuarantees 2024-04-14T11:15:52,873 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-04-14T11:15:52,873 INFO [PEWorker-2 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-04-14T11:15:52,879 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-04-14T11:15:52,897 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=4dbfc9ae90980452da73dfe0910874fc, REOPEN/MOVE}] 2024-04-14T11:15:52,898 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(729): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=4dbfc9ae90980452da73dfe0910874fc, REOPEN/MOVE 2024-04-14T11:15:52,899 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=4dbfc9ae90980452da73dfe0910874fc, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:52,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE; CloseRegionProcedure 4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:15:53,053 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,054 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(119): Close 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:53,055 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Closing 4dbfc9ae90980452da73dfe0910874fc, disabling compactions & flushes 2024-04-14T11:15:53,055 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1698): Closing region TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:53,055 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1719): Waiting without time limit for close lock on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:53,055 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1786): Acquired close lock on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. after waiting 0 ms 2024-04-14T11:15:53,055 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1796): Updates disabled for region TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:53,060 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-04-14T11:15:53,060 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1917): Closed TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:53,061 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1630): Region close journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:15:53,061 WARN [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3777): Not adding moved region record: 4dbfc9ae90980452da73dfe0910874fc to self. 2024-04-14T11:15:53,063 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(163): Closed 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:53,063 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=4dbfc9ae90980452da73dfe0910874fc, regionState=CLOSED 2024-04-14T11:15:53,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=40, resume processing ppid=39 2024-04-14T11:15:53,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=40, ppid=39, state=SUCCESS; CloseRegionProcedure 4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 in 163 msec 2024-04-14T11:15:53,067 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4dbfc9ae90980452da73dfe0910874fc, REOPEN/MOVE; state=CLOSED, location=jenkins-hbase6.apache.org,33239,1713093321027; forceNewPlan=false, retain=true 2024-04-14T11:15:53,218 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=4dbfc9ae90980452da73dfe0910874fc, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE; OpenRegionProcedure 4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:15:53,372 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,377 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:53,377 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7267): Opening region: {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} 2024-04-14T11:15:53,377 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:53,377 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(891): Instantiated TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:15:53,378 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7309): checking encryption for 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:53,378 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7312): checking classloading for 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:53,380 INFO [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:53,388 INFO [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dbfc9ae90980452da73dfe0910874fc columnFamilyName A 2024-04-14T11:15:53,390 DEBUG [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:53,391 INFO [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] regionserver.HStore(324): Store=4dbfc9ae90980452da73dfe0910874fc/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:15:53,391 INFO [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:53,392 INFO [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dbfc9ae90980452da73dfe0910874fc columnFamilyName B 2024-04-14T11:15:53,392 DEBUG [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:53,393 INFO [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] regionserver.HStore(324): Store=4dbfc9ae90980452da73dfe0910874fc/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:15:53,393 INFO [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:53,394 INFO [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4dbfc9ae90980452da73dfe0910874fc columnFamilyName C 2024-04-14T11:15:53,394 DEBUG [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:15:53,394 INFO [StoreOpener-4dbfc9ae90980452da73dfe0910874fc-1 {}] regionserver.HStore(324): Store=4dbfc9ae90980452da73dfe0910874fc/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:15:53,395 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:53,397 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:53,399 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-04-14T11:15:53,400 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1082): writing seq id for 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:53,401 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1099): Opened 4dbfc9ae90980452da73dfe0910874fc; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65805094, jitterRate=-0.01942768692970276}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-04-14T11:15:53,402 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(998): Region open journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:15:53,403 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2593): Post open deploy tasks for TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., pid=41, masterSystemTime=1713093353372 2024-04-14T11:15:53,405 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2620): Finished post open deploy task for TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:53,405 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:53,405 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=4dbfc9ae90980452da73dfe0910874fc, regionState=OPEN, openSeqNum=5, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=41, resume processing ppid=39 2024-04-14T11:15:53,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=41, ppid=39, state=SUCCESS; OpenRegionProcedure 4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 in 187 msec 2024-04-14T11:15:53,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=39, resume processing ppid=38 2024-04-14T11:15:53,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=39, ppid=38, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=4dbfc9ae90980452da73dfe0910874fc, REOPEN/MOVE in 511 msec 2024-04-14T11:15:53,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=38, resume processing ppid=37 2024-04-14T11:15:53,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=38, ppid=37, state=SUCCESS; ReopenTableRegionsProcedure table=TestAcidGuarantees in 535 msec 2024-04-14T11:15:53,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=37, state=SUCCESS; ModifyTableProcedure table=TestAcidGuarantees in 967 msec 2024-04-14T11:15:53,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=37 2024-04-14T11:15:53,429 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6365e97d to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:53,435 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@373667ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:53,437 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x36e1cef1 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:53,443 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ece6f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:53,444 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x579b618d to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:53,450 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@295d6e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:53,452 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1ec395f1 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:53,457 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f358f97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:53,459 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0fb426c5 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:53,465 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d5a46df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:53,467 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x62a21f25 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:53,474 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d11d110, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:53,475 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7a25463a to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:53,482 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@88c356c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:53,483 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7e2c3dae to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:53,488 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ee8a0c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:53,490 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4b6d84f7 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:15:53,494 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54331179, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:15:53,510 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:15:53,511 DEBUG [hconnection-0xd692a36-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-04-14T11:15:53,511 DEBUG [hconnection-0x3f12027f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:53,512 DEBUG [hconnection-0x55b448a4-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:53,512 DEBUG [hconnection-0x24bea844-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:53,520 DEBUG [hconnection-0x2129e838-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:53,520 DEBUG [hconnection-0x1580f395-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2024-04-14T11:15:53,520 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:37328, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:53,521 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:37350, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:53,524 DEBUG [hconnection-0x35c6648-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:53,524 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:37346, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:53,524 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:37344, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:53,524 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:37368, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:53,524 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:37358, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:53,525 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:15:53,525 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:37374, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:53,526 DEBUG [hconnection-0x428529cb-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:53,527 DEBUG [hconnection-0x5292db42-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:15:53,530 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:15:53,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:15:53,534 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:37378, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:53,543 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:37388, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:15:53,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:53,560 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:53,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-04-14T11:15:53,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2024-04-14T11:15:53,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:53,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093413613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:53,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093413613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:53,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093413615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093413616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093413625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404140a9921bec4dc4c3e945393e3660144a8_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093353560/Put/seqid=0 2024-04-14T11:15:53,681 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404140a9921bec4dc4c3e945393e3660144a8_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404140a9921bec4dc4c3e945393e3660144a8_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:53,683 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/7b83cca8b5034935a236e192f1e69425, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:15:53,692 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-04-14T11:15:53,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:53,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:15:53,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:53,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(53): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:53,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:53,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:53,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/7b83cca8b5034935a236e192f1e69425 is 175, key is test_row_0/A:col10/1713093353560/Put/seqid=0 2024-04-14T11:15:53,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:53,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093413736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:53,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093413737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093413737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093413737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093413737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2024-04-14T11:15:53,858 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-04-14T11:15:53,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:53,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:15:53,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:53,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(53): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:53,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:53,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:53,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093413946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093413946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093413947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093413948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:53,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093413951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,013 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-04-14T11:15:54,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:54,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:15:54,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:54,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(53): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:54,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2024-04-14T11:15:54,142 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-04-14T11:15:54,145 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/7b83cca8b5034935a236e192f1e69425 2024-04-14T11:15:54,167 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-04-14T11:15:54,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:54,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:15:54,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:54,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(53): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:54,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:54,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/7bd6b94bcee143e88a481d3790046d86 is 50, key is test_row_0/B:col10/1713093353560/Put/seqid=0 2024-04-14T11:15:54,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/7bd6b94bcee143e88a481d3790046d86 2024-04-14T11:15:54,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093414250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093414251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093414252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093414252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093414255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/d2eb2bc257de458482f60fadd297664d is 50, key is test_row_0/C:col10/1713093353560/Put/seqid=0 2024-04-14T11:15:54,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/d2eb2bc257de458482f60fadd297664d 2024-04-14T11:15:54,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/7b83cca8b5034935a236e192f1e69425 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/7b83cca8b5034935a236e192f1e69425 2024-04-14T11:15:54,321 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-04-14T11:15:54,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:54,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:15:54,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:54,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(53): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:54,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:54,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/7b83cca8b5034935a236e192f1e69425, entries=200, sequenceid=17, filesize=38.6 K 2024-04-14T11:15:54,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/7bd6b94bcee143e88a481d3790046d86 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/7bd6b94bcee143e88a481d3790046d86 2024-04-14T11:15:54,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/7bd6b94bcee143e88a481d3790046d86, entries=150, sequenceid=17, filesize=11.7 K 2024-04-14T11:15:54,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/d2eb2bc257de458482f60fadd297664d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/d2eb2bc257de458482f60fadd297664d 2024-04-14T11:15:54,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/d2eb2bc257de458482f60fadd297664d, entries=150, sequenceid=17, filesize=11.7 K 2024-04-14T11:15:54,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 4dbfc9ae90980452da73dfe0910874fc in 790ms, sequenceid=17, compaction requested=false 2024-04-14T11:15:54,350 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-04-14T11:15:54,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:15:54,474 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-04-14T11:15:54,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:54,475 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-04-14T11:15:54,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414f56af5548c324c6abc45e3427f6f31eb_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093353611/Put/seqid=0 2024-04-14T11:15:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2024-04-14T11:15:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:54,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:15:54,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:54,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093414766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093414767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093414767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093414769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093414765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093414871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093414874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093414876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093414877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093414881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:54,931 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414f56af5548c324c6abc45e3427f6f31eb_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414f56af5548c324c6abc45e3427f6f31eb_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:54,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/161ea00aa69c403bb5536e4aa672f77f, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:15:54,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/161ea00aa69c403bb5536e4aa672f77f is 175, key is test_row_0/A:col10/1713093353611/Put/seqid=0 2024-04-14T11:15:55,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093415076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:55,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093415080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:55,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093415080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:55,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093415081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:55,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093415087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:55,382 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/161ea00aa69c403bb5536e4aa672f77f 2024-04-14T11:15:55,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:55,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093415380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:55,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:55,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093415383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:55,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093415389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:55,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093415393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:55,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/27b75dd1902849d1a82a9a9b942583cc is 50, key is test_row_0/B:col10/1713093353611/Put/seqid=0 2024-04-14T11:15:55,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093415406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:55,438 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/27b75dd1902849d1a82a9a9b942583cc 2024-04-14T11:15:55,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/4514842d859b48c2a75624da615aea56 is 50, key is test_row_0/C:col10/1713093353611/Put/seqid=0 2024-04-14T11:15:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2024-04-14T11:15:55,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093415885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:55,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093415889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:55,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093415893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:55,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093415899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:55,916 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/4514842d859b48c2a75624da615aea56 2024-04-14T11:15:55,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093415916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:55,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/161ea00aa69c403bb5536e4aa672f77f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/161ea00aa69c403bb5536e4aa672f77f 2024-04-14T11:15:55,933 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/161ea00aa69c403bb5536e4aa672f77f, entries=150, sequenceid=41, filesize=30.2 K 2024-04-14T11:15:55,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/27b75dd1902849d1a82a9a9b942583cc as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/27b75dd1902849d1a82a9a9b942583cc 2024-04-14T11:15:55,942 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/27b75dd1902849d1a82a9a9b942583cc, entries=150, sequenceid=41, filesize=11.7 K 2024-04-14T11:15:55,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/4514842d859b48c2a75624da615aea56 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/4514842d859b48c2a75624da615aea56 2024-04-14T11:15:55,953 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/4514842d859b48c2a75624da615aea56, entries=150, sequenceid=41, filesize=11.7 K 2024-04-14T11:15:55,955 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3035): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4dbfc9ae90980452da73dfe0910874fc in 1480ms, sequenceid=41, compaction requested=false 2024-04-14T11:15:55,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2533): Flush status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:15:55,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:55,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=43 2024-04-14T11:15:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=43 2024-04-14T11:15:55,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=43, resume processing ppid=42 2024-04-14T11:15:55,966 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=43, ppid=42, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4260 sec 2024-04-14T11:15:55,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=42, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.4570 sec 2024-04-14T11:15:56,500 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-04-14T11:15:56,508 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:37572, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-04-14T11:15:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:56,894 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:56,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-04-14T11:15:56,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404140b1061a98c164662a322d9ee0f69cee2_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093356892/Put/seqid=0 2024-04-14T11:15:56,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093416937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:56,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093416937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:56,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093416939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:56,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093416941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:56,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093416944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:57,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093417046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:57,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093417047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:57,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093417047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:57,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093417049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:57,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093417050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:57,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:57,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093417251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093417251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:57,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093417251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:57,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093417254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:57,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093417254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:57,374 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404140b1061a98c164662a322d9ee0f69cee2_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404140b1061a98c164662a322d9ee0f69cee2_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:57,378 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/3a580da4b7cd48849047429c7a368189, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:15:57,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/3a580da4b7cd48849047429c7a368189 is 175, key is test_row_0/A:col10/1713093356892/Put/seqid=0 2024-04-14T11:15:57,417 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/3a580da4b7cd48849047429c7a368189 2024-04-14T11:15:57,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/2d6f1f2e3c9f4066b536c92df6774af4 is 50, key is test_row_0/B:col10/1713093356892/Put/seqid=0 2024-04-14T11:15:57,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/2d6f1f2e3c9f4066b536c92df6774af4 2024-04-14T11:15:57,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/4cacec1671fb4590b249acda23d6b1e2 is 50, key is test_row_0/C:col10/1713093356892/Put/seqid=0 2024-04-14T11:15:57,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/4cacec1671fb4590b249acda23d6b1e2 2024-04-14T11:15:57,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/3a580da4b7cd48849047429c7a368189 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3a580da4b7cd48849047429c7a368189 2024-04-14T11:15:57,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3a580da4b7cd48849047429c7a368189, entries=200, sequenceid=54, filesize=38.6 K 2024-04-14T11:15:57,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/2d6f1f2e3c9f4066b536c92df6774af4 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/2d6f1f2e3c9f4066b536c92df6774af4 2024-04-14T11:15:57,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/2d6f1f2e3c9f4066b536c92df6774af4, entries=150, sequenceid=54, filesize=11.7 K 2024-04-14T11:15:57,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/4cacec1671fb4590b249acda23d6b1e2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/4cacec1671fb4590b249acda23d6b1e2 2024-04-14T11:15:57,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093417556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:57,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093417556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:57,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093417556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:57,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093417558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:57,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:57,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093417559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:57,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/4cacec1671fb4590b249acda23d6b1e2, entries=150, sequenceid=54, filesize=11.7 K 2024-04-14T11:15:57,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 4dbfc9ae90980452da73dfe0910874fc in 672ms, sequenceid=54, compaction requested=true 2024-04-14T11:15:57,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:15:57,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:15:57,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:57,566 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:57,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:15:57,567 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:57,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:57,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:15:57,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:57,569 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:57,569 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/B is initiating minor compaction (all files) 2024-04-14T11:15:57,569 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/B in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:57,569 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/7bd6b94bcee143e88a481d3790046d86, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/27b75dd1902849d1a82a9a9b942583cc, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/2d6f1f2e3c9f4066b536c92df6774af4] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=35.1 K 2024-04-14T11:15:57,569 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110002 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:57,569 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/A is initiating minor compaction (all files) 2024-04-14T11:15:57,570 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/A in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:57,570 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/7b83cca8b5034935a236e192f1e69425, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/161ea00aa69c403bb5536e4aa672f77f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3a580da4b7cd48849047429c7a368189] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=107.4 K 2024-04-14T11:15:57,570 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:57,570 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/7b83cca8b5034935a236e192f1e69425, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/161ea00aa69c403bb5536e4aa672f77f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3a580da4b7cd48849047429c7a368189] 2024-04-14T11:15:57,571 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 7bd6b94bcee143e88a481d3790046d86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1713093353551 2024-04-14T11:15:57,571 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 27b75dd1902849d1a82a9a9b942583cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1713093353611 2024-04-14T11:15:57,571 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7b83cca8b5034935a236e192f1e69425, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1713093353551 2024-04-14T11:15:57,572 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 2d6f1f2e3c9f4066b536c92df6774af4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1713093354765 2024-04-14T11:15:57,573 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 161ea00aa69c403bb5536e4aa672f77f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1713093353611 2024-04-14T11:15:57,574 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3a580da4b7cd48849047429c7a368189, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1713093354762 2024-04-14T11:15:57,595 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#B#compaction#139 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:57,595 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/1a4f42a668c743f6937f51dc72670dc0 is 50, key is test_row_0/B:col10/1713093356892/Put/seqid=0 2024-04-14T11:15:57,617 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:15:57,630 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202404140c9bf47103cd49239ef59eeacd7ceeaf_4dbfc9ae90980452da73dfe0910874fc store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:15:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=42 2024-04-14T11:15:57,631 INFO [Thread-715 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 42 completed 2024-04-14T11:15:57,633 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:15:57,634 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202404140c9bf47103cd49239ef59eeacd7ceeaf_4dbfc9ae90980452da73dfe0910874fc, store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:15:57,635 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404140c9bf47103cd49239ef59eeacd7ceeaf_4dbfc9ae90980452da73dfe0910874fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:15:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-04-14T11:15:57,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2024-04-14T11:15:57,654 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:15:57,655 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:15:57,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:15:57,704 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#A#compaction#140 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:57,709 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/60e2be28a35f452a8ecf3412cf25580f is 175, key is test_row_0/A:col10/1713093356892/Put/seqid=0 2024-04-14T11:15:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2024-04-14T11:15:57,746 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/60e2be28a35f452a8ecf3412cf25580f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/60e2be28a35f452a8ecf3412cf25580f 2024-04-14T11:15:57,759 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/A of 4dbfc9ae90980452da73dfe0910874fc into 60e2be28a35f452a8ecf3412cf25580f(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:57,759 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:15:57,759 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/A, priority=13, startTime=1713093357566; duration=0sec 2024-04-14T11:15:57,759 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:57,759 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:A 2024-04-14T11:15:57,759 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:57,764 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:57,764 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/C is initiating minor compaction (all files) 2024-04-14T11:15:57,765 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/C in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:57,765 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/d2eb2bc257de458482f60fadd297664d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/4514842d859b48c2a75624da615aea56, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/4cacec1671fb4590b249acda23d6b1e2] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=35.1 K 2024-04-14T11:15:57,765 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting d2eb2bc257de458482f60fadd297664d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1713093353551 2024-04-14T11:15:57,766 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4514842d859b48c2a75624da615aea56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1713093353611 2024-04-14T11:15:57,767 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4cacec1671fb4590b249acda23d6b1e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1713093354765 2024-04-14T11:15:57,780 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:57,781 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/e913bc46e95c4807a237d89c497e3d72 is 50, key is test_row_0/C:col10/1713093356892/Put/seqid=0 2024-04-14T11:15:57,816 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/e913bc46e95c4807a237d89c497e3d72 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/e913bc46e95c4807a237d89c497e3d72 2024-04-14T11:15:57,826 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:57,826 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/C of 4dbfc9ae90980452da73dfe0910874fc into e913bc46e95c4807a237d89c497e3d72(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:57,826 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:15:57,827 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/C, priority=13, startTime=1713093357567; duration=0sec 2024-04-14T11:15:57,827 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:57,827 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:C 2024-04-14T11:15:57,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-04-14T11:15:57,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:57,827 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-04-14T11:15:57,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041489fb0a40a38e4d08811af8d46f3d3896_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093356938/Put/seqid=0 2024-04-14T11:15:57,881 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041489fb0a40a38e4d08811af8d46f3d3896_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041489fb0a40a38e4d08811af8d46f3d3896_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:57,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/8e53f493c09841478dfb6f372a8ed804, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:15:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/8e53f493c09841478dfb6f372a8ed804 is 175, key is test_row_0/A:col10/1713093356938/Put/seqid=0 2024-04-14T11:15:57,903 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/8e53f493c09841478dfb6f372a8ed804 2024-04-14T11:15:57,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/05296ede409a4dde891c0a2717c2b06a is 50, key is test_row_0/B:col10/1713093356938/Put/seqid=0 2024-04-14T11:15:57,940 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/05296ede409a4dde891c0a2717c2b06a 2024-04-14T11:15:57,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2024-04-14T11:15:57,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/93439a0a8aad4e6cabca5b509be497ee is 50, key is test_row_0/C:col10/1713093356938/Put/seqid=0 2024-04-14T11:15:58,004 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/93439a0a8aad4e6cabca5b509be497ee 2024-04-14T11:15:58,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/8e53f493c09841478dfb6f372a8ed804 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/8e53f493c09841478dfb6f372a8ed804 2024-04-14T11:15:58,028 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/8e53f493c09841478dfb6f372a8ed804, entries=150, sequenceid=79, filesize=30.2 K 2024-04-14T11:15:58,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/05296ede409a4dde891c0a2717c2b06a as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/05296ede409a4dde891c0a2717c2b06a 2024-04-14T11:15:58,038 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/05296ede409a4dde891c0a2717c2b06a, entries=150, sequenceid=79, filesize=11.7 K 2024-04-14T11:15:58,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/93439a0a8aad4e6cabca5b509be497ee as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/93439a0a8aad4e6cabca5b509be497ee 2024-04-14T11:15:58,047 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/93439a0a8aad4e6cabca5b509be497ee, entries=150, sequenceid=79, filesize=11.7 K 2024-04-14T11:15:58,048 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3035): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 4dbfc9ae90980452da73dfe0910874fc in 221ms, sequenceid=79, compaction requested=false 2024-04-14T11:15:58,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2533): Flush status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:15:58,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:58,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=45 2024-04-14T11:15:58,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=45 2024-04-14T11:15:58,051 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=45, resume processing ppid=44 2024-04-14T11:15:58,051 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=45, ppid=44, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 395 msec 2024-04-14T11:15:58,053 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=44, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 419 msec 2024-04-14T11:15:58,123 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/1a4f42a668c743f6937f51dc72670dc0 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/1a4f42a668c743f6937f51dc72670dc0 2024-04-14T11:15:58,135 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/B of 4dbfc9ae90980452da73dfe0910874fc into 1a4f42a668c743f6937f51dc72670dc0(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:58,135 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:15:58,135 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/B, priority=13, startTime=1713093357566; duration=0sec 2024-04-14T11:15:58,135 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:58,135 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:B 2024-04-14T11:15:58,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:58,241 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:58,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-04-14T11:15:58,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2024-04-14T11:15:58,244 INFO [Thread-715 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 44 completed 2024-04-14T11:15:58,246 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:15:58,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-04-14T11:15:58,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=46 2024-04-14T11:15:58,250 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:15:58,251 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:15:58,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:15:58,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414338ba9b6eb0e4a7695f3f6f5e0229c7b_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093358240/Put/seqid=0 2024-04-14T11:15:58,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093418308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093418308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093418309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093418310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093418312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,338 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414338ba9b6eb0e4a7695f3f6f5e0229c7b_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414338ba9b6eb0e4a7695f3f6f5e0229c7b_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:58,340 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/caa965d687f44fbcb12adc7de15c00eb, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:15:58,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/caa965d687f44fbcb12adc7de15c00eb is 175, key is test_row_0/A:col10/1713093358240/Put/seqid=0 2024-04-14T11:15:58,346 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/caa965d687f44fbcb12adc7de15c00eb 2024-04-14T11:15:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=46 2024-04-14T11:15:58,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/dee38c379cdb4c9a891bf6da8751e2c3 is 50, key is test_row_0/B:col10/1713093358240/Put/seqid=0 2024-04-14T11:15:58,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/dee38c379cdb4c9a891bf6da8751e2c3 2024-04-14T11:15:58,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/e737e028157d47c4a6990930b2519cb2 is 50, key is test_row_0/C:col10/1713093358240/Put/seqid=0 2024-04-14T11:15:58,403 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-04-14T11:15:58,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:58,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:15:58,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:58,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(53): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:58,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:58,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/e737e028157d47c4a6990930b2519cb2 2024-04-14T11:15:58,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/caa965d687f44fbcb12adc7de15c00eb as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/caa965d687f44fbcb12adc7de15c00eb 2024-04-14T11:15:58,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093418422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093418424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093418425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093418425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093418426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/caa965d687f44fbcb12adc7de15c00eb, entries=450, sequenceid=92, filesize=80.6 K 2024-04-14T11:15:58,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/dee38c379cdb4c9a891bf6da8751e2c3 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/dee38c379cdb4c9a891bf6da8751e2c3 2024-04-14T11:15:58,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/dee38c379cdb4c9a891bf6da8751e2c3, entries=450, sequenceid=92, filesize=25.4 K 2024-04-14T11:15:58,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/e737e028157d47c4a6990930b2519cb2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/e737e028157d47c4a6990930b2519cb2 2024-04-14T11:15:58,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/e737e028157d47c4a6990930b2519cb2, entries=450, sequenceid=92, filesize=25.4 K 2024-04-14T11:15:58,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4dbfc9ae90980452da73dfe0910874fc in 208ms, sequenceid=92, compaction requested=true 2024-04-14T11:15:58,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:15:58,450 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:58,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:15:58,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:58,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:15:58,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:58,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:15:58,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:58,450 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:58,451 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 144547 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:58,451 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/A is initiating minor compaction (all files) 2024-04-14T11:15:58,452 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/A in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:58,452 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/60e2be28a35f452a8ecf3412cf25580f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/8e53f493c09841478dfb6f372a8ed804, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/caa965d687f44fbcb12adc7de15c00eb] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=141.2 K 2024-04-14T11:15:58,452 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:58,452 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/60e2be28a35f452a8ecf3412cf25580f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/8e53f493c09841478dfb6f372a8ed804, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/caa965d687f44fbcb12adc7de15c00eb] 2024-04-14T11:15:58,452 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 50099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:58,452 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/B is initiating minor compaction (all files) 2024-04-14T11:15:58,452 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/B in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:58,452 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/1a4f42a668c743f6937f51dc72670dc0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/05296ede409a4dde891c0a2717c2b06a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/dee38c379cdb4c9a891bf6da8751e2c3] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=48.9 K 2024-04-14T11:15:58,452 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 60e2be28a35f452a8ecf3412cf25580f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1713093354765 2024-04-14T11:15:58,453 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 1a4f42a668c743f6937f51dc72670dc0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1713093354765 2024-04-14T11:15:58,454 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 05296ede409a4dde891c0a2717c2b06a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1713093356937 2024-04-14T11:15:58,454 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8e53f493c09841478dfb6f372a8ed804, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1713093356937 2024-04-14T11:15:58,454 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting dee38c379cdb4c9a891bf6da8751e2c3, keycount=450, bloomtype=ROW, size=25.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1713093358174 2024-04-14T11:15:58,455 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting caa965d687f44fbcb12adc7de15c00eb, keycount=450, bloomtype=ROW, size=80.6 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1713093358174 2024-04-14T11:15:58,469 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#B#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:58,469 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/d608047cdeb04fef831e0d0f770276db is 50, key is test_row_0/B:col10/1713093358240/Put/seqid=0 2024-04-14T11:15:58,470 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:15:58,473 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024041461512b7d6aa5442f82ae345c1ad65944_4dbfc9ae90980452da73dfe0910874fc store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:15:58,477 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024041461512b7d6aa5442f82ae345c1ad65944_4dbfc9ae90980452da73dfe0910874fc, store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:15:58,477 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041461512b7d6aa5442f82ae345c1ad65944_4dbfc9ae90980452da73dfe0910874fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:15:58,508 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#A#compaction#149 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:58,509 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/bb0287312c084a89a4dd44abd71164eb is 175, key is test_row_0/A:col10/1713093358240/Put/seqid=0 2024-04-14T11:15:58,511 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/d608047cdeb04fef831e0d0f770276db as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/d608047cdeb04fef831e0d0f770276db 2024-04-14T11:15:58,518 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/B of 4dbfc9ae90980452da73dfe0910874fc into d608047cdeb04fef831e0d0f770276db(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:58,518 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:15:58,518 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/B, priority=13, startTime=1713093358450; duration=0sec 2024-04-14T11:15:58,518 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:15:58,518 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:B 2024-04-14T11:15:58,518 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:15:58,521 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 50099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:15:58,521 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/C is initiating minor compaction (all files) 2024-04-14T11:15:58,521 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/C in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:58,521 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/e913bc46e95c4807a237d89c497e3d72, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/93439a0a8aad4e6cabca5b509be497ee, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/e737e028157d47c4a6990930b2519cb2] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=48.9 K 2024-04-14T11:15:58,522 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting e913bc46e95c4807a237d89c497e3d72, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1713093354765 2024-04-14T11:15:58,523 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 93439a0a8aad4e6cabca5b509be497ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1713093356937 2024-04-14T11:15:58,524 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting e737e028157d47c4a6990930b2519cb2, keycount=450, bloomtype=ROW, size=25.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1713093358174 2024-04-14T11:15:58,535 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/bb0287312c084a89a4dd44abd71164eb as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/bb0287312c084a89a4dd44abd71164eb 2024-04-14T11:15:58,541 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#C#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:15:58,542 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/ec40a0f30d7e4184b627a607fe0e4386 is 50, key is test_row_0/C:col10/1713093358240/Put/seqid=0 2024-04-14T11:15:58,543 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/A of 4dbfc9ae90980452da73dfe0910874fc into bb0287312c084a89a4dd44abd71164eb(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:58,543 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:15:58,543 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/A, priority=13, startTime=1713093358449; duration=0sec 2024-04-14T11:15:58,543 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:58,543 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:A 2024-04-14T11:15:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=46 2024-04-14T11:15:58,558 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-04-14T11:15:58,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:58,559 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-04-14T11:15:58,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404140ab4e8f2b1c1407baf5579b70c40338d_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093358307/Put/seqid=0 2024-04-14T11:15:58,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:58,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:15:58,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093418635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093418635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093418640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093418641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093418641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093418743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093418743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093418743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093418744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093418744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=46 2024-04-14T11:15:58,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093418946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093418946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093418946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093418947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:58,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093418947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:58,956 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/ec40a0f30d7e4184b627a607fe0e4386 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/ec40a0f30d7e4184b627a607fe0e4386 2024-04-14T11:15:58,964 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/C of 4dbfc9ae90980452da73dfe0910874fc into ec40a0f30d7e4184b627a607fe0e4386(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:15:58,964 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:15:58,965 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/C, priority=13, startTime=1713093358450; duration=0sec 2024-04-14T11:15:58,965 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:15:58,965 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:C 2024-04-14T11:15:58,982 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404140ab4e8f2b1c1407baf5579b70c40338d_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404140ab4e8f2b1c1407baf5579b70c40338d_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:58,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/3da9bd332f0c45dfbd947eea8882b329, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:15:58,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/3da9bd332f0c45dfbd947eea8882b329 is 175, key is test_row_0/A:col10/1713093358307/Put/seqid=0 2024-04-14T11:15:58,991 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/3da9bd332f0c45dfbd947eea8882b329 2024-04-14T11:15:59,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/13a26856f28c4e74aeebacdd1effc56f is 50, key is test_row_0/B:col10/1713093358307/Put/seqid=0 2024-04-14T11:15:59,010 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/13a26856f28c4e74aeebacdd1effc56f 2024-04-14T11:15:59,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/1cbf5b5d10074761b1d940372b3eb958 is 50, key is test_row_0/C:col10/1713093358307/Put/seqid=0 2024-04-14T11:15:59,049 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/1cbf5b5d10074761b1d940372b3eb958 2024-04-14T11:15:59,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/3da9bd332f0c45dfbd947eea8882b329 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3da9bd332f0c45dfbd947eea8882b329 2024-04-14T11:15:59,064 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3da9bd332f0c45dfbd947eea8882b329, entries=150, sequenceid=118, filesize=30.2 K 2024-04-14T11:15:59,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/13a26856f28c4e74aeebacdd1effc56f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/13a26856f28c4e74aeebacdd1effc56f 2024-04-14T11:15:59,073 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/13a26856f28c4e74aeebacdd1effc56f, entries=150, sequenceid=118, filesize=11.7 K 2024-04-14T11:15:59,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/1cbf5b5d10074761b1d940372b3eb958 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/1cbf5b5d10074761b1d940372b3eb958 2024-04-14T11:15:59,083 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/1cbf5b5d10074761b1d940372b3eb958, entries=150, sequenceid=118, filesize=11.7 K 2024-04-14T11:15:59,085 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3035): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 4dbfc9ae90980452da73dfe0910874fc in 526ms, sequenceid=118, compaction requested=false 2024-04-14T11:15:59,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2533): Flush status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:15:59,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:59,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=47 2024-04-14T11:15:59,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=47 2024-04-14T11:15:59,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=47, resume processing ppid=46 2024-04-14T11:15:59,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=47, ppid=46, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 835 msec 2024-04-14T11:15:59,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=46, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 843 msec 2024-04-14T11:15:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:59,251 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:15:59,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-04-14T11:15:59,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414148d27e835ae4711a420a500bce47b56_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093359249/Put/seqid=0 2024-04-14T11:15:59,282 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414148d27e835ae4711a420a500bce47b56_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414148d27e835ae4711a420a500bce47b56_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:15:59,283 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/b0fa2a4f72bb46ad97edbd5818e7cb0c, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:15:59,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/b0fa2a4f72bb46ad97edbd5818e7cb0c is 175, key is test_row_0/A:col10/1713093359249/Put/seqid=0 2024-04-14T11:15:59,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093419278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093419280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093419281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093419282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093419283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=46 2024-04-14T11:15:59,355 INFO [Thread-715 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 46 completed 2024-04-14T11:15:59,357 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:15:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-04-14T11:15:59,362 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:15:59,363 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:15:59,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:15:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2024-04-14T11:15:59,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093419386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093419386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093419386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093419388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093419390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2024-04-14T11:15:59,515 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-04-14T11:15:59,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:59,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:15:59,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:59,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(53): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:59,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:59,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093419589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093419590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093419590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093419591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093419593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2024-04-14T11:15:59,668 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-04-14T11:15:59,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:59,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:15:59,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:59,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(53): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:59,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:59,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:59,695 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/b0fa2a4f72bb46ad97edbd5818e7cb0c 2024-04-14T11:15:59,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/25a0cba62b164ee3a658c93516f58452 is 50, key is test_row_0/B:col10/1713093359249/Put/seqid=0 2024-04-14T11:15:59,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/25a0cba62b164ee3a658c93516f58452 2024-04-14T11:15:59,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/54d2d1972cfd4b9caf0f69c41439edf9 is 50, key is test_row_0/C:col10/1713093359249/Put/seqid=0 2024-04-14T11:15:59,821 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-04-14T11:15:59,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:59,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:15:59,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:59,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(53): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:59,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:59,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093419893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093419894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093419894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093419894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:15:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093419899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2024-04-14T11:15:59,975 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:15:59,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-04-14T11:15:59,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:59,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:15:59,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:15:59,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(53): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:59,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:15:59,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:00,128 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:00,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-04-14T11:16:00,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:00,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:00,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:00,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(53): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:00,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:00,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/54d2d1972cfd4b9caf0f69c41439edf9 2024-04-14T11:16:00,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/b0fa2a4f72bb46ad97edbd5818e7cb0c as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/b0fa2a4f72bb46ad97edbd5818e7cb0c 2024-04-14T11:16:00,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/b0fa2a4f72bb46ad97edbd5818e7cb0c, entries=200, sequenceid=133, filesize=38.8 K 2024-04-14T11:16:00,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/25a0cba62b164ee3a658c93516f58452 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/25a0cba62b164ee3a658c93516f58452 2024-04-14T11:16:00,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/25a0cba62b164ee3a658c93516f58452, entries=150, sequenceid=133, filesize=11.8 K 2024-04-14T11:16:00,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/54d2d1972cfd4b9caf0f69c41439edf9 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/54d2d1972cfd4b9caf0f69c41439edf9 2024-04-14T11:16:00,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/54d2d1972cfd4b9caf0f69c41439edf9, entries=150, sequenceid=133, filesize=11.8 K 2024-04-14T11:16:00,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 4dbfc9ae90980452da73dfe0910874fc in 921ms, sequenceid=133, compaction requested=true 2024-04-14T11:16:00,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:00,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:00,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:00,173 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:00,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:00,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:00,173 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:00,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:00,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:00,174 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101764 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:00,175 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/A is initiating minor compaction (all files) 2024-04-14T11:16:00,175 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/A in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:00,175 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/bb0287312c084a89a4dd44abd71164eb, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3da9bd332f0c45dfbd947eea8882b329, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/b0fa2a4f72bb46ad97edbd5818e7cb0c] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=99.4 K 2024-04-14T11:16:00,175 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:00,175 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/bb0287312c084a89a4dd44abd71164eb, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3da9bd332f0c45dfbd947eea8882b329, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/b0fa2a4f72bb46ad97edbd5818e7cb0c] 2024-04-14T11:16:00,175 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36308 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:00,175 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/B is initiating minor compaction (all files) 2024-04-14T11:16:00,175 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/B in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:00,175 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/d608047cdeb04fef831e0d0f770276db, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/13a26856f28c4e74aeebacdd1effc56f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/25a0cba62b164ee3a658c93516f58452] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=35.5 K 2024-04-14T11:16:00,176 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting bb0287312c084a89a4dd44abd71164eb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1713093358238 2024-04-14T11:16:00,177 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3da9bd332f0c45dfbd947eea8882b329, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1713093358305 2024-04-14T11:16:00,177 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting d608047cdeb04fef831e0d0f770276db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1713093358238 2024-04-14T11:16:00,177 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting b0fa2a4f72bb46ad97edbd5818e7cb0c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1713093358633 2024-04-14T11:16:00,178 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 13a26856f28c4e74aeebacdd1effc56f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1713093358305 2024-04-14T11:16:00,180 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 25a0cba62b164ee3a658c93516f58452, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1713093358639 2024-04-14T11:16:00,193 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:00,194 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#B#compaction#157 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:00,195 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/dffec1432ebb4a2bb64d5e053ceacec8 is 50, key is test_row_0/B:col10/1713093359249/Put/seqid=0 2024-04-14T11:16:00,201 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20240414542de8bf2a0b4b67888d0ac95ee38995_4dbfc9ae90980452da73dfe0910874fc store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:00,203 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20240414542de8bf2a0b4b67888d0ac95ee38995_4dbfc9ae90980452da73dfe0910874fc, store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:00,204 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414542de8bf2a0b4b67888d0ac95ee38995_4dbfc9ae90980452da73dfe0910874fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:00,258 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#A#compaction#158 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:00,259 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/c0d64ab28ab3427496728995bbc84661 is 175, key is test_row_0/A:col10/1713093359249/Put/seqid=0 2024-04-14T11:16:00,276 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/c0d64ab28ab3427496728995bbc84661 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/c0d64ab28ab3427496728995bbc84661 2024-04-14T11:16:00,282 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:00,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-04-14T11:16:00,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:00,283 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-04-14T11:16:00,286 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/A of 4dbfc9ae90980452da73dfe0910874fc into c0d64ab28ab3427496728995bbc84661(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:00,286 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:00,286 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/A, priority=13, startTime=1713093360173; duration=0sec 2024-04-14T11:16:00,286 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:00,287 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:A 2024-04-14T11:16:00,287 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:00,294 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36308 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:00,294 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/C is initiating minor compaction (all files) 2024-04-14T11:16:00,294 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/C in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:00,294 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/ec40a0f30d7e4184b627a607fe0e4386, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/1cbf5b5d10074761b1d940372b3eb958, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/54d2d1972cfd4b9caf0f69c41439edf9] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=35.5 K 2024-04-14T11:16:00,296 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting ec40a0f30d7e4184b627a607fe0e4386, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1713093358238 2024-04-14T11:16:00,297 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1cbf5b5d10074761b1d940372b3eb958, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1713093358305 2024-04-14T11:16:00,297 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 54d2d1972cfd4b9caf0f69c41439edf9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1713093358639 2024-04-14T11:16:00,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414524ff5d0499f4ee8b2ef4a61ef4ca48f_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093359282/Put/seqid=0 2024-04-14T11:16:00,314 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#C#compaction#160 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:00,315 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/a36c5cd0f66b4d0d9af59549cc677d02 is 50, key is test_row_0/C:col10/1713093359249/Put/seqid=0 2024-04-14T11:16:00,350 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/a36c5cd0f66b4d0d9af59549cc677d02 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/a36c5cd0f66b4d0d9af59549cc677d02 2024-04-14T11:16:00,358 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/C of 4dbfc9ae90980452da73dfe0910874fc into a36c5cd0f66b4d0d9af59549cc677d02(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:00,359 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:00,359 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/C, priority=13, startTime=1713093360173; duration=0sec 2024-04-14T11:16:00,359 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:00,359 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:C 2024-04-14T11:16:00,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:00,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:00,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093420415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:00,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093420436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:00,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093420436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:00,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093420436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:00,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093420437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2024-04-14T11:16:00,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:00,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093420539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:00,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:00,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093420542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093420542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:00,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093420543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:00,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093420543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:00,640 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/dffec1432ebb4a2bb64d5e053ceacec8 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/dffec1432ebb4a2bb64d5e053ceacec8 2024-04-14T11:16:00,646 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/B of 4dbfc9ae90980452da73dfe0910874fc into dffec1432ebb4a2bb64d5e053ceacec8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:00,647 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:00,647 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/B, priority=13, startTime=1713093360173; duration=0sec 2024-04-14T11:16:00,647 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:00,647 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:B 2024-04-14T11:16:00,707 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-04-14T11:16:00,707 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-04-14T11:16:00,726 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414524ff5d0499f4ee8b2ef4a61ef4ca48f_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414524ff5d0499f4ee8b2ef4a61ef4ca48f_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:00,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/c89a4cb1eddb4898ad269c3715d35c18, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:00,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/c89a4cb1eddb4898ad269c3715d35c18 is 175, key is test_row_0/A:col10/1713093359282/Put/seqid=0 2024-04-14T11:16:00,733 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/c89a4cb1eddb4898ad269c3715d35c18 2024-04-14T11:16:00,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093420742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:00,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/4bc27bcef64f4675abe0a657415df6e7 is 50, key is test_row_0/B:col10/1713093359282/Put/seqid=0 2024-04-14T11:16:00,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:00,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093420745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093420745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:00,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093420745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:00,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093420746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:01,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093421044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:01,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093421047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:01,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093421047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:01,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093421049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:01,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093421050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:01,150 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/4bc27bcef64f4675abe0a657415df6e7 2024-04-14T11:16:01,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/8a344fbb41714817b21dc12d0d17fb15 is 50, key is test_row_0/C:col10/1713093359282/Put/seqid=0 2024-04-14T11:16:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2024-04-14T11:16:01,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093421548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:01,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093421549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:01,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093421552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:01,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093421555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:01,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093421555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:01,567 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/8a344fbb41714817b21dc12d0d17fb15 2024-04-14T11:16:01,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/c89a4cb1eddb4898ad269c3715d35c18 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/c89a4cb1eddb4898ad269c3715d35c18 2024-04-14T11:16:01,581 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/c89a4cb1eddb4898ad269c3715d35c18, entries=150, sequenceid=155, filesize=30.4 K 2024-04-14T11:16:01,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/4bc27bcef64f4675abe0a657415df6e7 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/4bc27bcef64f4675abe0a657415df6e7 2024-04-14T11:16:01,588 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/4bc27bcef64f4675abe0a657415df6e7, entries=150, sequenceid=155, filesize=11.8 K 2024-04-14T11:16:01,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/8a344fbb41714817b21dc12d0d17fb15 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/8a344fbb41714817b21dc12d0d17fb15 2024-04-14T11:16:01,598 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/8a344fbb41714817b21dc12d0d17fb15, entries=150, sequenceid=155, filesize=11.8 K 2024-04-14T11:16:01,599 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3035): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 4dbfc9ae90980452da73dfe0910874fc in 1316ms, sequenceid=155, compaction requested=false 2024-04-14T11:16:01,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2533): Flush status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:01,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:01,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=49 2024-04-14T11:16:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=49 2024-04-14T11:16:01,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=49, resume processing ppid=48 2024-04-14T11:16:01,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=49, ppid=48, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2380 sec 2024-04-14T11:16:01,605 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=48, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.2460 sec 2024-04-14T11:16:02,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:02,555 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:02,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-04-14T11:16:02,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414dd3fc829690443d89390a3b84695f141_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093362554/Put/seqid=0 2024-04-14T11:16:02,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:02,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093422576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:02,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:02,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093422577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:02,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:02,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093422577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:02,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:02,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093422578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:02,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:02,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093422579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:02,584 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414dd3fc829690443d89390a3b84695f141_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414dd3fc829690443d89390a3b84695f141_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:02,585 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/9f53502d158b4d5bb8d21df468f10984, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:02,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/9f53502d158b4d5bb8d21df468f10984 is 175, key is test_row_0/A:col10/1713093362554/Put/seqid=0 2024-04-14T11:16:02,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093422681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:02,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:02,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093422681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:02,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093422681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:02,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:02,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093422683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:02,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:02,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093422683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:02,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:02,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093422883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:02,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093422884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:02,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093422884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:02,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:02,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093422886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:02,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:02,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093422886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:02,993 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/9f53502d158b4d5bb8d21df468f10984 2024-04-14T11:16:03,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/f872e92202734c7b8ad070c0c5c69ec6 is 50, key is test_row_0/B:col10/1713093362554/Put/seqid=0 2024-04-14T11:16:03,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/f872e92202734c7b8ad070c0c5c69ec6 2024-04-14T11:16:03,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/8b2a5cbbddcd4930be15c1fcef476e2d is 50, key is test_row_0/C:col10/1713093362554/Put/seqid=0 2024-04-14T11:16:03,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/8b2a5cbbddcd4930be15c1fcef476e2d 2024-04-14T11:16:03,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/9f53502d158b4d5bb8d21df468f10984 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/9f53502d158b4d5bb8d21df468f10984 2024-04-14T11:16:03,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/9f53502d158b4d5bb8d21df468f10984, entries=200, sequenceid=173, filesize=38.8 K 2024-04-14T11:16:03,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/f872e92202734c7b8ad070c0c5c69ec6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/f872e92202734c7b8ad070c0c5c69ec6 2024-04-14T11:16:03,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/f872e92202734c7b8ad070c0c5c69ec6, entries=150, sequenceid=173, filesize=11.8 K 2024-04-14T11:16:03,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/8b2a5cbbddcd4930be15c1fcef476e2d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/8b2a5cbbddcd4930be15c1fcef476e2d 2024-04-14T11:16:03,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/8b2a5cbbddcd4930be15c1fcef476e2d, entries=150, sequenceid=173, filesize=11.8 K 2024-04-14T11:16:03,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 4dbfc9ae90980452da73dfe0910874fc in 500ms, sequenceid=173, compaction requested=true 2024-04-14T11:16:03,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:03,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:03,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:03,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:03,055 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:03,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:03,055 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:03,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:03,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:03,056 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102216 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:03,057 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/A is initiating minor compaction (all files) 2024-04-14T11:16:03,057 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/A in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:03,057 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36710 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:03,057 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/c0d64ab28ab3427496728995bbc84661, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/c89a4cb1eddb4898ad269c3715d35c18, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/9f53502d158b4d5bb8d21df468f10984] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=99.8 K 2024-04-14T11:16:03,057 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/B is initiating minor compaction (all files) 2024-04-14T11:16:03,057 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:03,057 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/B in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:03,057 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/c0d64ab28ab3427496728995bbc84661, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/c89a4cb1eddb4898ad269c3715d35c18, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/9f53502d158b4d5bb8d21df468f10984] 2024-04-14T11:16:03,057 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/dffec1432ebb4a2bb64d5e053ceacec8, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/4bc27bcef64f4675abe0a657415df6e7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/f872e92202734c7b8ad070c0c5c69ec6] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=35.8 K 2024-04-14T11:16:03,058 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c0d64ab28ab3427496728995bbc84661, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1713093358639 2024-04-14T11:16:03,058 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting dffec1432ebb4a2bb64d5e053ceacec8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1713093358639 2024-04-14T11:16:03,058 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c89a4cb1eddb4898ad269c3715d35c18, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1713093359279 2024-04-14T11:16:03,059 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 4bc27bcef64f4675abe0a657415df6e7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1713093359279 2024-04-14T11:16:03,059 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9f53502d158b4d5bb8d21df468f10984, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1713093360401 2024-04-14T11:16:03,060 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting f872e92202734c7b8ad070c0c5c69ec6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1713093360401 2024-04-14T11:16:03,077 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:03,079 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#B#compaction#166 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:03,079 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/1bff09597ce340a8a092e45f69d4f77d is 50, key is test_row_0/B:col10/1713093362554/Put/seqid=0 2024-04-14T11:16:03,080 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20240414e0b1bae50edf492ba5233a8c8c00a8c3_4dbfc9ae90980452da73dfe0910874fc store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:03,082 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20240414e0b1bae50edf492ba5233a8c8c00a8c3_4dbfc9ae90980452da73dfe0910874fc, store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:03,082 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414e0b1bae50edf492ba5233a8c8c00a8c3_4dbfc9ae90980452da73dfe0910874fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:03,105 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#A#compaction#167 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:03,105 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/6956369671494167b8d530ccdc1692ff is 175, key is test_row_0/A:col10/1713093362554/Put/seqid=0 2024-04-14T11:16:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:03,188 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:03,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-04-14T11:16:03,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404140797e940ec8b49da94b90a6b06fde953_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093362572/Put/seqid=0 2024-04-14T11:16:03,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093423197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093423198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093423198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093423199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093423201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093423303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093423303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093423303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093423304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093423305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=48 2024-04-14T11:16:03,470 INFO [Thread-715 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 48 completed 2024-04-14T11:16:03,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-04-14T11:16:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2024-04-14T11:16:03,475 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:03,476 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:03,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:03,502 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/1bff09597ce340a8a092e45f69d4f77d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/1bff09597ce340a8a092e45f69d4f77d 2024-04-14T11:16:03,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093423505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093423505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093423506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093423508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093423509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,511 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/B of 4dbfc9ae90980452da73dfe0910874fc into 1bff09597ce340a8a092e45f69d4f77d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:03,511 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:03,511 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/B, priority=13, startTime=1713093363055; duration=0sec 2024-04-14T11:16:03,511 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:03,511 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:B 2024-04-14T11:16:03,511 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:03,513 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36710 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:03,513 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/C is initiating minor compaction (all files) 2024-04-14T11:16:03,513 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/C in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:03,513 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/a36c5cd0f66b4d0d9af59549cc677d02, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/8a344fbb41714817b21dc12d0d17fb15, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/8b2a5cbbddcd4930be15c1fcef476e2d] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=35.8 K 2024-04-14T11:16:03,514 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting a36c5cd0f66b4d0d9af59549cc677d02, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1713093358639 2024-04-14T11:16:03,514 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 8a344fbb41714817b21dc12d0d17fb15, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1713093359279 2024-04-14T11:16:03,515 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 8b2a5cbbddcd4930be15c1fcef476e2d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1713093360401 2024-04-14T11:16:03,520 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/6956369671494167b8d530ccdc1692ff as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/6956369671494167b8d530ccdc1692ff 2024-04-14T11:16:03,526 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#C#compaction#169 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:03,526 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/8889e23432f24182b68213b88a4f6f35 is 50, key is test_row_0/C:col10/1713093362554/Put/seqid=0 2024-04-14T11:16:03,528 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/A of 4dbfc9ae90980452da73dfe0910874fc into 6956369671494167b8d530ccdc1692ff(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:03,528 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:03,528 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/A, priority=13, startTime=1713093363055; duration=0sec 2024-04-14T11:16:03,528 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:03,528 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:A 2024-04-14T11:16:03,547 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/8889e23432f24182b68213b88a4f6f35 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/8889e23432f24182b68213b88a4f6f35 2024-04-14T11:16:03,553 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/C of 4dbfc9ae90980452da73dfe0910874fc into 8889e23432f24182b68213b88a4f6f35(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:03,553 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:03,553 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/C, priority=13, startTime=1713093363055; duration=0sec 2024-04-14T11:16:03,554 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:03,554 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:C 2024-04-14T11:16:03,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2024-04-14T11:16:03,608 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404140797e940ec8b49da94b90a6b06fde953_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404140797e940ec8b49da94b90a6b06fde953_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:03,609 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/1e5fe4875c6c46c7933c5b59a61b15cb, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:03,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/1e5fe4875c6c46c7933c5b59a61b15cb is 175, key is test_row_0/A:col10/1713093362572/Put/seqid=0 2024-04-14T11:16:03,615 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/1e5fe4875c6c46c7933c5b59a61b15cb 2024-04-14T11:16:03,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/67e9a8cbc3ea49919845b4a12292f2ff is 50, key is test_row_0/B:col10/1713093362572/Put/seqid=0 2024-04-14T11:16:03,627 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-04-14T11:16:03,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:03,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:03,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:03,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(53): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:03,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/67e9a8cbc3ea49919845b4a12292f2ff 2024-04-14T11:16:03,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:03,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/7c11c926ffe74ce990c14b2fb0751fc6 is 50, key is test_row_0/C:col10/1713093362572/Put/seqid=0 2024-04-14T11:16:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2024-04-14T11:16:03,780 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-04-14T11:16:03,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:03,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:03,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:03,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(53): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:03,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:03,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093423810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093423810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093423812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093423813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093423813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,934 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:03,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-04-14T11:16:03,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:03,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:03,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:03,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(53): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:03,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:03,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:04,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/7c11c926ffe74ce990c14b2fb0751fc6 2024-04-14T11:16:04,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/1e5fe4875c6c46c7933c5b59a61b15cb as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/1e5fe4875c6c46c7933c5b59a61b15cb 2024-04-14T11:16:04,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/1e5fe4875c6c46c7933c5b59a61b15cb, entries=200, sequenceid=196, filesize=38.8 K 2024-04-14T11:16:04,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/67e9a8cbc3ea49919845b4a12292f2ff as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/67e9a8cbc3ea49919845b4a12292f2ff 2024-04-14T11:16:04,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/67e9a8cbc3ea49919845b4a12292f2ff, entries=150, sequenceid=196, filesize=11.8 K 2024-04-14T11:16:04,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/7c11c926ffe74ce990c14b2fb0751fc6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/7c11c926ffe74ce990c14b2fb0751fc6 2024-04-14T11:16:04,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/7c11c926ffe74ce990c14b2fb0751fc6, entries=150, sequenceid=196, filesize=11.8 K 2024-04-14T11:16:04,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 4dbfc9ae90980452da73dfe0910874fc in 885ms, sequenceid=196, compaction requested=false 2024-04-14T11:16:04,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2024-04-14T11:16:04,087 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-04-14T11:16:04,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:04,088 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-04-14T11:16:04,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414b40e19a0fbd44102b787ca8ad44ec342_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093363198/Put/seqid=0 2024-04-14T11:16:04,104 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414b40e19a0fbd44102b787ca8ad44ec342_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414b40e19a0fbd44102b787ca8ad44ec342_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:04,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/4424a0c467b549cab7125d860e88904a, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:04,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/4424a0c467b549cab7125d860e88904a is 175, key is test_row_0/A:col10/1713093363198/Put/seqid=0 2024-04-14T11:16:04,120 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/4424a0c467b549cab7125d860e88904a 2024-04-14T11:16:04,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/8b82ce200a454e00be95174a080d52d5 is 50, key is test_row_0/B:col10/1713093363198/Put/seqid=0 2024-04-14T11:16:04,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:04,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:04,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093424330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093424332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093424332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093424333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093424334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093424435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093424437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093424437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093424437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093424437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,538 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/8b82ce200a454e00be95174a080d52d5 2024-04-14T11:16:04,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/5cff079c87a547fd92590a9da1ea3969 is 50, key is test_row_0/C:col10/1713093363198/Put/seqid=0 2024-04-14T11:16:04,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2024-04-14T11:16:04,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093424638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093424640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093424641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093424641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093424642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,750 INFO [master/jenkins-hbase6:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-04-14T11:16:04,750 INFO [master/jenkins-hbase6:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-04-14T11:16:04,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093424940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093424945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093424946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093424950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:04,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093424950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:04,952 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/5cff079c87a547fd92590a9da1ea3969 2024-04-14T11:16:04,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/4424a0c467b549cab7125d860e88904a as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/4424a0c467b549cab7125d860e88904a 2024-04-14T11:16:04,965 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/4424a0c467b549cab7125d860e88904a, entries=150, sequenceid=212, filesize=30.4 K 2024-04-14T11:16:04,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/8b82ce200a454e00be95174a080d52d5 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/8b82ce200a454e00be95174a080d52d5 2024-04-14T11:16:04,972 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/8b82ce200a454e00be95174a080d52d5, entries=150, sequenceid=212, filesize=11.8 K 2024-04-14T11:16:04,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/5cff079c87a547fd92590a9da1ea3969 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/5cff079c87a547fd92590a9da1ea3969 2024-04-14T11:16:04,979 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/5cff079c87a547fd92590a9da1ea3969, entries=150, sequenceid=212, filesize=11.8 K 2024-04-14T11:16:04,980 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3035): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 4dbfc9ae90980452da73dfe0910874fc in 893ms, sequenceid=212, compaction requested=true 2024-04-14T11:16:04,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2533): Flush status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:04,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:04,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=51 2024-04-14T11:16:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=51 2024-04-14T11:16:04,983 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=51, resume processing ppid=50 2024-04-14T11:16:04,983 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=51, ppid=50, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5060 sec 2024-04-14T11:16:04,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=50, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.5120 sec 2024-04-14T11:16:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:05,444 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:05,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-04-14T11:16:05,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041424fc14d0c65f4415a6707f7b9b866d20_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093365443/Put/seqid=0 2024-04-14T11:16:05,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093425455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:05,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093425458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:05,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093425460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:05,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093425461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:05,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093425461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:05,466 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041424fc14d0c65f4415a6707f7b9b866d20_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041424fc14d0c65f4415a6707f7b9b866d20_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:05,468 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/38004244c086437881a76be0610f8025, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:05,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/38004244c086437881a76be0610f8025 is 175, key is test_row_0/A:col10/1713093365443/Put/seqid=0 2024-04-14T11:16:05,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093425562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:05,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093425562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:05,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093425562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:05,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093425564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:05,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093425565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2024-04-14T11:16:05,581 INFO [Thread-715 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 50 completed 2024-04-14T11:16:05,583 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-04-14T11:16:05,587 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2024-04-14T11:16:05,587 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:05,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2024-04-14T11:16:05,739 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:05,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-04-14T11:16:05,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:05,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:05,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:05,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(53): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:05,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:05,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:05,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093425764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093425764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:05,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093425765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:05,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093425767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:05,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093425768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:05,875 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/38004244c086437881a76be0610f8025 2024-04-14T11:16:05,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/e0c6cdf10d17451daa981752c5bb2140 is 50, key is test_row_0/B:col10/1713093365443/Put/seqid=0 2024-04-14T11:16:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2024-04-14T11:16:05,892 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:05,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-04-14T11:16:05,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:05,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:05,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:05,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(53): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:05,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:06,045 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:06,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-04-14T11:16:06,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:06,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:06,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:06,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(53): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:06,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:06,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093426066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:06,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093426067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:06,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093426068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:06,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093426072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:06,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093426072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2024-04-14T11:16:06,199 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:06,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-04-14T11:16:06,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:06,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:06,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:06,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(53): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:06,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:06,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:06,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/e0c6cdf10d17451daa981752c5bb2140 2024-04-14T11:16:06,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/e997bc7721b049f3bf53048743d08e4b is 50, key is test_row_0/C:col10/1713093365443/Put/seqid=0 2024-04-14T11:16:06,352 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:06,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-04-14T11:16:06,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:06,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:06,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:06,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(53): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:06,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:06,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:06,505 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:06,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-04-14T11:16:06,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:06,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:06,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:06,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(53): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:06,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:06,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:06,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093426571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:06,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093426572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:06,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093426582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:06,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093426583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:06,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:06,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093426583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:06,658 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:06,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-04-14T11:16:06,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:06,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:06,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:06,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(53): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:06,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:06,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2024-04-14T11:16:06,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/e997bc7721b049f3bf53048743d08e4b 2024-04-14T11:16:06,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/38004244c086437881a76be0610f8025 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/38004244c086437881a76be0610f8025 2024-04-14T11:16:06,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/38004244c086437881a76be0610f8025, entries=200, sequenceid=237, filesize=38.8 K 2024-04-14T11:16:06,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/e0c6cdf10d17451daa981752c5bb2140 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/e0c6cdf10d17451daa981752c5bb2140 2024-04-14T11:16:06,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/e0c6cdf10d17451daa981752c5bb2140, entries=150, sequenceid=237, filesize=11.8 K 2024-04-14T11:16:06,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/e997bc7721b049f3bf53048743d08e4b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/e997bc7721b049f3bf53048743d08e4b 2024-04-14T11:16:06,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/e997bc7721b049f3bf53048743d08e4b, entries=150, sequenceid=237, filesize=11.8 K 2024-04-14T11:16:06,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 4dbfc9ae90980452da73dfe0910874fc in 1302ms, sequenceid=237, compaction requested=true 2024-04-14T11:16:06,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:06,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:06,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:06,747 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:06,747 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:06,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:06,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:06,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:06,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:06,750 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142050 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:06,750 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/A is initiating minor compaction (all files) 2024-04-14T11:16:06,750 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:06,750 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/A in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:06,750 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/B is initiating minor compaction (all files) 2024-04-14T11:16:06,750 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/6956369671494167b8d530ccdc1692ff, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/1e5fe4875c6c46c7933c5b59a61b15cb, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/4424a0c467b549cab7125d860e88904a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/38004244c086437881a76be0610f8025] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=138.7 K 2024-04-14T11:16:06,750 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/B in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:06,750 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:06,750 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/1bff09597ce340a8a092e45f69d4f77d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/67e9a8cbc3ea49919845b4a12292f2ff, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/8b82ce200a454e00be95174a080d52d5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/e0c6cdf10d17451daa981752c5bb2140] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=47.8 K 2024-04-14T11:16:06,750 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/6956369671494167b8d530ccdc1692ff, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/1e5fe4875c6c46c7933c5b59a61b15cb, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/4424a0c467b549cab7125d860e88904a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/38004244c086437881a76be0610f8025] 2024-04-14T11:16:06,751 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 1bff09597ce340a8a092e45f69d4f77d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1713093360401 2024-04-14T11:16:06,752 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6956369671494167b8d530ccdc1692ff, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1713093360401 2024-04-14T11:16:06,753 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 67e9a8cbc3ea49919845b4a12292f2ff, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1713093362572 2024-04-14T11:16:06,753 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1e5fe4875c6c46c7933c5b59a61b15cb, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1713093362572 2024-04-14T11:16:06,754 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 8b82ce200a454e00be95174a080d52d5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1713093363197 2024-04-14T11:16:06,754 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4424a0c467b549cab7125d860e88904a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1713093363197 2024-04-14T11:16:06,754 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting e0c6cdf10d17451daa981752c5bb2140, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1713093364323 2024-04-14T11:16:06,755 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 38004244c086437881a76be0610f8025, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1713093364323 2024-04-14T11:16:06,775 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#B#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:06,776 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/c5b9d60bd46d4a88a246992798e995a1 is 50, key is test_row_0/B:col10/1713093365443/Put/seqid=0 2024-04-14T11:16:06,781 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:06,805 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20240414a95b4805c5204b9db147d99f36fd43f9_4dbfc9ae90980452da73dfe0910874fc store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:06,808 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20240414a95b4805c5204b9db147d99f36fd43f9_4dbfc9ae90980452da73dfe0910874fc, store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:06,808 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414a95b4805c5204b9db147d99f36fd43f9_4dbfc9ae90980452da73dfe0910874fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:06,811 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:06,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-04-14T11:16:06,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:06,813 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-04-14T11:16:06,843 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/c5b9d60bd46d4a88a246992798e995a1 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/c5b9d60bd46d4a88a246992798e995a1 2024-04-14T11:16:06,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414c013e2087f864f9b97ad474fe8f44821_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093365459/Put/seqid=0 2024-04-14T11:16:06,855 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#A#compaction#179 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:06,855 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/B of 4dbfc9ae90980452da73dfe0910874fc into c5b9d60bd46d4a88a246992798e995a1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:06,855 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:06,855 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/B, priority=12, startTime=1713093366747; duration=0sec 2024-04-14T11:16:06,858 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/cea404b346a44473832b9dec8cbf867f is 175, key is test_row_0/A:col10/1713093365443/Put/seqid=0 2024-04-14T11:16:06,859 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:06,859 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:B 2024-04-14T11:16:06,859 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:06,863 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:06,863 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/C is initiating minor compaction (all files) 2024-04-14T11:16:06,863 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/C in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:06,863 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/8889e23432f24182b68213b88a4f6f35, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/7c11c926ffe74ce990c14b2fb0751fc6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/5cff079c87a547fd92590a9da1ea3969, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/e997bc7721b049f3bf53048743d08e4b] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=47.8 K 2024-04-14T11:16:06,863 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 8889e23432f24182b68213b88a4f6f35, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1713093360401 2024-04-14T11:16:06,864 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 7c11c926ffe74ce990c14b2fb0751fc6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1713093362572 2024-04-14T11:16:06,865 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 5cff079c87a547fd92590a9da1ea3969, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1713093363197 2024-04-14T11:16:06,865 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting e997bc7721b049f3bf53048743d08e4b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1713093364323 2024-04-14T11:16:06,876 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#C#compaction#181 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:06,877 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/0879b9a7ebab4e3ba2d6e56470d6bd9b is 50, key is test_row_0/C:col10/1713093365443/Put/seqid=0 2024-04-14T11:16:06,886 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414c013e2087f864f9b97ad474fe8f44821_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414c013e2087f864f9b97ad474fe8f44821_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:06,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/836e02e1be14442aa2ad16b7b7c1116f, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:06,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/836e02e1be14442aa2ad16b7b7c1116f is 175, key is test_row_0/A:col10/1713093365459/Put/seqid=0 2024-04-14T11:16:06,896 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/836e02e1be14442aa2ad16b7b7c1116f 2024-04-14T11:16:06,900 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/0879b9a7ebab4e3ba2d6e56470d6bd9b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/0879b9a7ebab4e3ba2d6e56470d6bd9b 2024-04-14T11:16:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/35d7fbbc30774db7809ca4d5df3846a2 is 50, key is test_row_0/B:col10/1713093365459/Put/seqid=0 2024-04-14T11:16:06,913 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/C of 4dbfc9ae90980452da73dfe0910874fc into 0879b9a7ebab4e3ba2d6e56470d6bd9b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:06,913 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:06,913 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/C, priority=12, startTime=1713093366747; duration=0sec 2024-04-14T11:16:06,913 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:06,913 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:C 2024-04-14T11:16:06,914 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/35d7fbbc30774db7809ca4d5df3846a2 2024-04-14T11:16:06,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/39efda50c4724f909e7c0e27eb968d5a is 50, key is test_row_0/C:col10/1713093365459/Put/seqid=0 2024-04-14T11:16:07,276 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/cea404b346a44473832b9dec8cbf867f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/cea404b346a44473832b9dec8cbf867f 2024-04-14T11:16:07,283 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/A of 4dbfc9ae90980452da73dfe0910874fc into cea404b346a44473832b9dec8cbf867f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:07,283 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:07,283 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/A, priority=12, startTime=1713093366747; duration=0sec 2024-04-14T11:16:07,283 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:07,283 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:A 2024-04-14T11:16:07,336 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/39efda50c4724f909e7c0e27eb968d5a 2024-04-14T11:16:07,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/836e02e1be14442aa2ad16b7b7c1116f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/836e02e1be14442aa2ad16b7b7c1116f 2024-04-14T11:16:07,349 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/836e02e1be14442aa2ad16b7b7c1116f, entries=150, sequenceid=249, filesize=30.4 K 2024-04-14T11:16:07,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/35d7fbbc30774db7809ca4d5df3846a2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/35d7fbbc30774db7809ca4d5df3846a2 2024-04-14T11:16:07,357 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/35d7fbbc30774db7809ca4d5df3846a2, entries=150, sequenceid=249, filesize=11.8 K 2024-04-14T11:16:07,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/39efda50c4724f909e7c0e27eb968d5a as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/39efda50c4724f909e7c0e27eb968d5a 2024-04-14T11:16:07,367 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/39efda50c4724f909e7c0e27eb968d5a, entries=150, sequenceid=249, filesize=11.8 K 2024-04-14T11:16:07,368 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3035): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 4dbfc9ae90980452da73dfe0910874fc in 555ms, sequenceid=249, compaction requested=false 2024-04-14T11:16:07,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2533): Flush status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:07,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:07,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=53 2024-04-14T11:16:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=53 2024-04-14T11:16:07,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=53, resume processing ppid=52 2024-04-14T11:16:07,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=53, ppid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7820 sec 2024-04-14T11:16:07,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.7890 sec 2024-04-14T11:16:07,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:07,664 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:07,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-04-14T11:16:07,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414db502b3d83dd4fb28b4323a28cc981af_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093367659/Put/seqid=0 2024-04-14T11:16:07,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2024-04-14T11:16:07,693 INFO [Thread-715 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 52 completed 2024-04-14T11:16:07,693 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414db502b3d83dd4fb28b4323a28cc981af_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414db502b3d83dd4fb28b4323a28cc981af_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:07,696 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/00284f8f6b1a46cb9dabff8239377673, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:07,696 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:07,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/00284f8f6b1a46cb9dabff8239377673 is 175, key is test_row_0/A:col10/1713093367659/Put/seqid=0 2024-04-14T11:16:07,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-04-14T11:16:07,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2024-04-14T11:16:07,701 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:07,703 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/00284f8f6b1a46cb9dabff8239377673 2024-04-14T11:16:07,703 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:07,703 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:07,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:07,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093427704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093427702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:07,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:07,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093427706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:07,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093427707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:07,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:07,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093427711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:07,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/e8d4f73e01fc402b9484ea06092e6ba9 is 50, key is test_row_0/B:col10/1713093367659/Put/seqid=0 2024-04-14T11:16:07,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2024-04-14T11:16:07,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:07,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:07,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093427812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:07,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093427812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:07,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:07,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:07,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093427813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:07,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093427813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:07,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:07,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093427815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:07,855 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:07,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-04-14T11:16:07,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:07,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:07,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:07,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(53): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:07,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:07,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:08,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2024-04-14T11:16:08,008 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-04-14T11:16:08,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:08,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:08,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:08,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(53): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:08,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:08,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:08,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:08,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093428015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:08,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093428015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:08,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093428016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:08,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093428016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:08,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093428017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/e8d4f73e01fc402b9484ea06092e6ba9 2024-04-14T11:16:08,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/93036490485145e8906f8f80ef098e42 is 50, key is test_row_0/C:col10/1713093367659/Put/seqid=0 2024-04-14T11:16:08,162 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-04-14T11:16:08,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:08,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:08,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:08,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(53): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:08,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:08,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2024-04-14T11:16:08,315 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-04-14T11:16:08,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:08,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:08,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:08,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(53): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:08,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:08,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093428319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093428320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093428320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093428322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093428323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,470 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-04-14T11:16:08,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:08,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:08,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:08,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(53): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:08,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:08,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:08,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/93036490485145e8906f8f80ef098e42 2024-04-14T11:16:08,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/00284f8f6b1a46cb9dabff8239377673 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/00284f8f6b1a46cb9dabff8239377673 2024-04-14T11:16:08,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/00284f8f6b1a46cb9dabff8239377673, entries=400, sequenceid=263, filesize=73.0 K 2024-04-14T11:16:08,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/e8d4f73e01fc402b9484ea06092e6ba9 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/e8d4f73e01fc402b9484ea06092e6ba9 2024-04-14T11:16:08,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/e8d4f73e01fc402b9484ea06092e6ba9, entries=400, sequenceid=263, filesize=23.9 K 2024-04-14T11:16:08,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/93036490485145e8906f8f80ef098e42 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/93036490485145e8906f8f80ef098e42 2024-04-14T11:16:08,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/93036490485145e8906f8f80ef098e42, entries=150, sequenceid=263, filesize=12.0 K 2024-04-14T11:16:08,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4dbfc9ae90980452da73dfe0910874fc in 909ms, sequenceid=263, compaction requested=true 2024-04-14T11:16:08,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:08,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:08,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:08,574 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:08,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:08,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:08,574 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:08,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:08,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:08,579 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 49252 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:08,579 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/B is initiating minor compaction (all files) 2024-04-14T11:16:08,579 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/B in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:08,579 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 137450 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:08,579 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/c5b9d60bd46d4a88a246992798e995a1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/35d7fbbc30774db7809ca4d5df3846a2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/e8d4f73e01fc402b9484ea06092e6ba9] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=48.1 K 2024-04-14T11:16:08,579 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/A is initiating minor compaction (all files) 2024-04-14T11:16:08,579 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/A in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:08,579 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/cea404b346a44473832b9dec8cbf867f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/836e02e1be14442aa2ad16b7b7c1116f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/00284f8f6b1a46cb9dabff8239377673] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=134.2 K 2024-04-14T11:16:08,579 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:08,579 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/cea404b346a44473832b9dec8cbf867f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/836e02e1be14442aa2ad16b7b7c1116f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/00284f8f6b1a46cb9dabff8239377673] 2024-04-14T11:16:08,580 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting c5b9d60bd46d4a88a246992798e995a1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1713093364323 2024-04-14T11:16:08,580 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting cea404b346a44473832b9dec8cbf867f, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1713093364323 2024-04-14T11:16:08,581 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 35d7fbbc30774db7809ca4d5df3846a2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1713093365457 2024-04-14T11:16:08,581 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 836e02e1be14442aa2ad16b7b7c1116f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1713093365457 2024-04-14T11:16:08,581 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting e8d4f73e01fc402b9484ea06092e6ba9, keycount=400, bloomtype=ROW, size=23.9 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1713093367612 2024-04-14T11:16:08,581 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 00284f8f6b1a46cb9dabff8239377673, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1713093367612 2024-04-14T11:16:08,591 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:08,592 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#B#compaction#187 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:08,593 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/2a62a8ec48c04673b810bcc9f9fff216 is 50, key is test_row_0/B:col10/1713093367659/Put/seqid=0 2024-04-14T11:16:08,593 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202404149c38998c12644ed59be1af424b97297b_4dbfc9ae90980452da73dfe0910874fc store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:08,595 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202404149c38998c12644ed59be1af424b97297b_4dbfc9ae90980452da73dfe0910874fc, store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:08,596 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404149c38998c12644ed59be1af424b97297b_4dbfc9ae90980452da73dfe0910874fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:08,623 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-04-14T11:16:08,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:08,624 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-04-14T11:16:08,625 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#A#compaction#188 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:08,626 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/036b2d15ec284c54b4141ffc881f3767 is 175, key is test_row_0/A:col10/1713093367659/Put/seqid=0 2024-04-14T11:16:08,633 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/2a62a8ec48c04673b810bcc9f9fff216 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/2a62a8ec48c04673b810bcc9f9fff216 2024-04-14T11:16:08,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041485184cc896af4505b0b2add39fd3a3f4_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093367689/Put/seqid=0 2024-04-14T11:16:08,643 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/036b2d15ec284c54b4141ffc881f3767 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/036b2d15ec284c54b4141ffc881f3767 2024-04-14T11:16:08,644 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/B of 4dbfc9ae90980452da73dfe0910874fc into 2a62a8ec48c04673b810bcc9f9fff216(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:08,644 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:08,644 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/B, priority=13, startTime=1713093368574; duration=0sec 2024-04-14T11:16:08,644 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:08,644 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:B 2024-04-14T11:16:08,644 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:08,646 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37098 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:08,646 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/C is initiating minor compaction (all files) 2024-04-14T11:16:08,647 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/C in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:08,647 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/0879b9a7ebab4e3ba2d6e56470d6bd9b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/39efda50c4724f909e7c0e27eb968d5a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/93036490485145e8906f8f80ef098e42] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=36.2 K 2024-04-14T11:16:08,648 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 0879b9a7ebab4e3ba2d6e56470d6bd9b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1713093364323 2024-04-14T11:16:08,648 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 39efda50c4724f909e7c0e27eb968d5a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1713093365457 2024-04-14T11:16:08,650 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 93036490485145e8906f8f80ef098e42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1713093367659 2024-04-14T11:16:08,653 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/A of 4dbfc9ae90980452da73dfe0910874fc into 036b2d15ec284c54b4141ffc881f3767(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:08,653 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:08,653 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/A, priority=13, startTime=1713093368573; duration=0sec 2024-04-14T11:16:08,653 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:08,653 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:A 2024-04-14T11:16:08,655 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041485184cc896af4505b0b2add39fd3a3f4_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041485184cc896af4505b0b2add39fd3a3f4_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:08,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/ed9dd625fd834e8bb9351e937414ed87, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:08,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/ed9dd625fd834e8bb9351e937414ed87 is 175, key is test_row_0/A:col10/1713093367689/Put/seqid=0 2024-04-14T11:16:08,663 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#C#compaction#190 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:08,663 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/5fd1f9ba17ca4449b9ec8ebc3719b160 is 50, key is test_row_0/C:col10/1713093367659/Put/seqid=0 2024-04-14T11:16:08,685 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/5fd1f9ba17ca4449b9ec8ebc3719b160 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/5fd1f9ba17ca4449b9ec8ebc3719b160 2024-04-14T11:16:08,694 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/C of 4dbfc9ae90980452da73dfe0910874fc into 5fd1f9ba17ca4449b9ec8ebc3719b160(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:08,694 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:08,694 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/C, priority=13, startTime=1713093368574; duration=0sec 2024-04-14T11:16:08,694 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:08,694 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:C 2024-04-14T11:16:08,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2024-04-14T11:16:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:08,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:08,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093428828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093428829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093428829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093428830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093428833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093428932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093428933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093428934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:08,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093428937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:09,067 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/ed9dd625fd834e8bb9351e937414ed87 2024-04-14T11:16:09,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/bc71dfbfa442458783eb63730ff5c87c is 50, key is test_row_0/B:col10/1713093367689/Put/seqid=0 2024-04-14T11:16:09,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093429135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:09,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093429136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:09,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093429137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:09,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093429138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:09,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:09,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093429438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093429438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:09,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093429440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:09,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093429443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:09,493 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/bc71dfbfa442458783eb63730ff5c87c 2024-04-14T11:16:09,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/bd7129104ef54e158e2622767dff61bd is 50, key is test_row_0/C:col10/1713093367689/Put/seqid=0 2024-04-14T11:16:09,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2024-04-14T11:16:09,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093429838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:09,914 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/bd7129104ef54e158e2622767dff61bd 2024-04-14T11:16:09,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/ed9dd625fd834e8bb9351e937414ed87 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/ed9dd625fd834e8bb9351e937414ed87 2024-04-14T11:16:09,929 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/ed9dd625fd834e8bb9351e937414ed87, entries=150, sequenceid=288, filesize=30.5 K 2024-04-14T11:16:09,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/bc71dfbfa442458783eb63730ff5c87c as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/bc71dfbfa442458783eb63730ff5c87c 2024-04-14T11:16:09,937 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/bc71dfbfa442458783eb63730ff5c87c, entries=150, sequenceid=288, filesize=12.0 K 2024-04-14T11:16:09,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/bd7129104ef54e158e2622767dff61bd as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/bd7129104ef54e158e2622767dff61bd 2024-04-14T11:16:09,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:09,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093429943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:09,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093429944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:09,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093429944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:09,948 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/bd7129104ef54e158e2622767dff61bd, entries=150, sequenceid=288, filesize=12.0 K 2024-04-14T11:16:09,949 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3035): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 4dbfc9ae90980452da73dfe0910874fc in 1325ms, sequenceid=288, compaction requested=false 2024-04-14T11:16:09,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2533): Flush status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:09,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:09,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=55 2024-04-14T11:16:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=55 2024-04-14T11:16:09,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:09,951 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:09,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-04-14T11:16:09,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=55, resume processing ppid=54 2024-04-14T11:16:09,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=55, ppid=54, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2480 sec 2024-04-14T11:16:09,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=54, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.2580 sec 2024-04-14T11:16:09,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414dd411552a87643ecaebaad3fe572d2c2_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093369950/Put/seqid=0 2024-04-14T11:16:10,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093430026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:10,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093430131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:10,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093430334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:10,395 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414dd411552a87643ecaebaad3fe572d2c2_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414dd411552a87643ecaebaad3fe572d2c2_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:10,396 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/00f909093cb04963b4e744a5eb01442e, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:10,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/00f909093cb04963b4e744a5eb01442e is 175, key is test_row_0/A:col10/1713093369950/Put/seqid=0 2024-04-14T11:16:10,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093430636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:10,802 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/00f909093cb04963b4e744a5eb01442e 2024-04-14T11:16:10,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/db253e82adc341a292f4df3dfaddfd61 is 50, key is test_row_0/B:col10/1713093369950/Put/seqid=0 2024-04-14T11:16:10,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/db253e82adc341a292f4df3dfaddfd61 2024-04-14T11:16:10,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/cde070c111144a6481d167c305c265c6 is 50, key is test_row_0/C:col10/1713093369950/Put/seqid=0 2024-04-14T11:16:10,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093430948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:10,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093430948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:10,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093430949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:11,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093431145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:11,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/cde070c111144a6481d167c305c265c6 2024-04-14T11:16:11,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/00f909093cb04963b4e744a5eb01442e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/00f909093cb04963b4e744a5eb01442e 2024-04-14T11:16:11,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/00f909093cb04963b4e744a5eb01442e, entries=200, sequenceid=303, filesize=39.0 K 2024-04-14T11:16:11,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/db253e82adc341a292f4df3dfaddfd61 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/db253e82adc341a292f4df3dfaddfd61 2024-04-14T11:16:11,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/db253e82adc341a292f4df3dfaddfd61, entries=150, sequenceid=303, filesize=12.0 K 2024-04-14T11:16:11,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/cde070c111144a6481d167c305c265c6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/cde070c111144a6481d167c305c265c6 2024-04-14T11:16:11,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/cde070c111144a6481d167c305c265c6, entries=150, sequenceid=303, filesize=12.0 K 2024-04-14T11:16:11,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4dbfc9ae90980452da73dfe0910874fc in 1324ms, sequenceid=303, compaction requested=true 2024-04-14T11:16:11,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:11,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:11,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:11,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:11,276 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:11,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:11,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:11,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-04-14T11:16:11,276 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:11,279 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37500 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:11,279 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/B is initiating minor compaction (all files) 2024-04-14T11:16:11,280 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/B in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:11,280 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/2a62a8ec48c04673b810bcc9f9fff216, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/bc71dfbfa442458783eb63730ff5c87c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/db253e82adc341a292f4df3dfaddfd61] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=36.6 K 2024-04-14T11:16:11,280 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103056 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:11,280 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/A is initiating minor compaction (all files) 2024-04-14T11:16:11,281 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/A in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:11,281 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/036b2d15ec284c54b4141ffc881f3767, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/ed9dd625fd834e8bb9351e937414ed87, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/00f909093cb04963b4e744a5eb01442e] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=100.6 K 2024-04-14T11:16:11,281 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:11,281 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/036b2d15ec284c54b4141ffc881f3767, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/ed9dd625fd834e8bb9351e937414ed87, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/00f909093cb04963b4e744a5eb01442e] 2024-04-14T11:16:11,281 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 2a62a8ec48c04673b810bcc9f9fff216, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1713093367659 2024-04-14T11:16:11,282 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 036b2d15ec284c54b4141ffc881f3767, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1713093367659 2024-04-14T11:16:11,283 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting bc71dfbfa442458783eb63730ff5c87c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1713093367689 2024-04-14T11:16:11,283 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting ed9dd625fd834e8bb9351e937414ed87, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1713093367689 2024-04-14T11:16:11,283 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting db253e82adc341a292f4df3dfaddfd61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1713093368824 2024-04-14T11:16:11,284 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 00f909093cb04963b4e744a5eb01442e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1713093368824 2024-04-14T11:16:11,314 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:11,330 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#B#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:11,331 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/2c87253d19234b1db59b7fe70128f550 is 50, key is test_row_0/B:col10/1713093369950/Put/seqid=0 2024-04-14T11:16:11,337 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20240414ce74f8b0313a418cab5ed3c5edce8f41_4dbfc9ae90980452da73dfe0910874fc store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:11,339 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20240414ce74f8b0313a418cab5ed3c5edce8f41_4dbfc9ae90980452da73dfe0910874fc, store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:11,339 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414ce74f8b0313a418cab5ed3c5edce8f41_4dbfc9ae90980452da73dfe0910874fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:11,415 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/2c87253d19234b1db59b7fe70128f550 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/2c87253d19234b1db59b7fe70128f550 2024-04-14T11:16:11,424 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/B of 4dbfc9ae90980452da73dfe0910874fc into 2c87253d19234b1db59b7fe70128f550(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:11,424 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:11,424 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/B, priority=13, startTime=1713093371276; duration=0sec 2024-04-14T11:16:11,424 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:11,424 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:B 2024-04-14T11:16:11,424 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:11,428 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37500 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:11,428 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/C is initiating minor compaction (all files) 2024-04-14T11:16:11,428 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/C in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:11,429 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/5fd1f9ba17ca4449b9ec8ebc3719b160, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/bd7129104ef54e158e2622767dff61bd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/cde070c111144a6481d167c305c265c6] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=36.6 K 2024-04-14T11:16:11,429 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 5fd1f9ba17ca4449b9ec8ebc3719b160, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1713093367659 2024-04-14T11:16:11,430 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting bd7129104ef54e158e2622767dff61bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1713093367689 2024-04-14T11:16:11,430 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting cde070c111144a6481d167c305c265c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1713093368824 2024-04-14T11:16:11,451 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#C#compaction#198 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:11,451 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/c44d49836f01402f8c1aa0c747482316 is 50, key is test_row_0/C:col10/1713093369950/Put/seqid=0 2024-04-14T11:16:11,498 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/c44d49836f01402f8c1aa0c747482316 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/c44d49836f01402f8c1aa0c747482316 2024-04-14T11:16:11,516 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/C of 4dbfc9ae90980452da73dfe0910874fc into c44d49836f01402f8c1aa0c747482316(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:11,516 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:11,517 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/C, priority=13, startTime=1713093371276; duration=0sec 2024-04-14T11:16:11,517 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:11,517 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:C 2024-04-14T11:16:11,799 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#A#compaction#196 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:11,800 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/3198069c742c4e2391999f5558496cbf is 175, key is test_row_0/A:col10/1713093369950/Put/seqid=0 2024-04-14T11:16:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2024-04-14T11:16:11,809 INFO [Thread-715 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 54 completed 2024-04-14T11:16:11,810 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-04-14T11:16:11,814 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2024-04-14T11:16:11,815 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:11,815 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:11,852 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:11,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-04-14T11:16:11,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414075ab0fcbf30489aae14c989edceedba_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093371851/Put/seqid=0 2024-04-14T11:16:11,870 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414075ab0fcbf30489aae14c989edceedba_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414075ab0fcbf30489aae14c989edceedba_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:11,871 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/ff6bd2158b89473caa5763286b1c0dee, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:11,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/ff6bd2158b89473caa5763286b1c0dee is 175, key is test_row_0/A:col10/1713093371851/Put/seqid=0 2024-04-14T11:16:11,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093431890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:11,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2024-04-14T11:16:11,966 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:11,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-04-14T11:16:11,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:11,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:11,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:11,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(53): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:11,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:11,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093431994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2024-04-14T11:16:12,120 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:12,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-04-14T11:16:12,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:12,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:12,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:12,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(53): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:12,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:12,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37388 deadline: 1713093432157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:12,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:12,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093432196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:12,215 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/3198069c742c4e2391999f5558496cbf as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3198069c742c4e2391999f5558496cbf 2024-04-14T11:16:12,225 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/A of 4dbfc9ae90980452da73dfe0910874fc into 3198069c742c4e2391999f5558496cbf(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:12,226 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:12,226 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/A, priority=13, startTime=1713093371276; duration=0sec 2024-04-14T11:16:12,226 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:12,226 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:A 2024-04-14T11:16:12,273 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:12,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-04-14T11:16:12,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:12,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:12,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:12,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(53): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:12,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:12,280 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/ff6bd2158b89473caa5763286b1c0dee 2024-04-14T11:16:12,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/569c3aa2fd99439bb5faa4131c00ba5e is 50, key is test_row_0/B:col10/1713093371851/Put/seqid=0 2024-04-14T11:16:12,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/569c3aa2fd99439bb5faa4131c00ba5e 2024-04-14T11:16:12,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/a34df326bc894cc28948252eb990f832 is 50, key is test_row_0/C:col10/1713093371851/Put/seqid=0 2024-04-14T11:16:12,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/a34df326bc894cc28948252eb990f832 2024-04-14T11:16:12,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/ff6bd2158b89473caa5763286b1c0dee as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/ff6bd2158b89473caa5763286b1c0dee 2024-04-14T11:16:12,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/ff6bd2158b89473caa5763286b1c0dee, entries=150, sequenceid=330, filesize=30.5 K 2024-04-14T11:16:12,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/569c3aa2fd99439bb5faa4131c00ba5e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/569c3aa2fd99439bb5faa4131c00ba5e 2024-04-14T11:16:12,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/569c3aa2fd99439bb5faa4131c00ba5e, entries=150, sequenceid=330, filesize=12.0 K 2024-04-14T11:16:12,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/a34df326bc894cc28948252eb990f832 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/a34df326bc894cc28948252eb990f832 2024-04-14T11:16:12,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/a34df326bc894cc28948252eb990f832, entries=150, sequenceid=330, filesize=12.0 K 2024-04-14T11:16:12,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 4dbfc9ae90980452da73dfe0910874fc in 517ms, sequenceid=330, compaction requested=false 2024-04-14T11:16:12,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2024-04-14T11:16:12,428 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:12,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-04-14T11:16:12,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:12,429 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-04-14T11:16:12,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404145799a70ab8b14675ac0cb8a85f4c0357_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093371881/Put/seqid=0 2024-04-14T11:16:12,479 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404145799a70ab8b14675ac0cb8a85f4c0357_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404145799a70ab8b14675ac0cb8a85f4c0357_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:12,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/3627b7f9726d48d6b8ce5e824fdc6758, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:12,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/3627b7f9726d48d6b8ce5e824fdc6758 is 175, key is test_row_0/A:col10/1713093371881/Put/seqid=0 2024-04-14T11:16:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:12,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. as already flushing 2024-04-14T11:16:12,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093432610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:12,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093432714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:12,906 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/3627b7f9726d48d6b8ce5e824fdc6758 2024-04-14T11:16:12,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2024-04-14T11:16:12,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093432920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:12,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/ae476eedf9e44be99a2a2212b7b9815d is 50, key is test_row_0/B:col10/1713093371881/Put/seqid=0 2024-04-14T11:16:12,952 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/ae476eedf9e44be99a2a2212b7b9815d 2024-04-14T11:16:12,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37346 deadline: 1713093432953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:12,956 DEBUG [Thread-713 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4124 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., hostname=jenkins-hbase6.apache.org,33239,1713093321027, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2024-04-14T11:16:12,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37358 deadline: 1713093432958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:12,962 DEBUG [Thread-709 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4131 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., hostname=jenkins-hbase6.apache.org,33239,1713093321027, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2024-04-14T11:16:12,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:12,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37328 deadline: 1713093432964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:12,966 DEBUG [Thread-711 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4138 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., hostname=jenkins-hbase6.apache.org,33239,1713093321027, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2024-04-14T11:16:12,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/7289ddefc3ce4df99c329fe42c95df60 is 50, key is test_row_0/C:col10/1713093371881/Put/seqid=0 2024-04-14T11:16:13,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:13,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:37344 deadline: 1713093433229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:13,380 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/7289ddefc3ce4df99c329fe42c95df60 2024-04-14T11:16:13,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/3627b7f9726d48d6b8ce5e824fdc6758 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3627b7f9726d48d6b8ce5e824fdc6758 2024-04-14T11:16:13,395 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3627b7f9726d48d6b8ce5e824fdc6758, entries=150, sequenceid=342, filesize=30.5 K 2024-04-14T11:16:13,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/ae476eedf9e44be99a2a2212b7b9815d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/ae476eedf9e44be99a2a2212b7b9815d 2024-04-14T11:16:13,404 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/ae476eedf9e44be99a2a2212b7b9815d, entries=150, sequenceid=342, filesize=12.0 K 2024-04-14T11:16:13,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/7289ddefc3ce4df99c329fe42c95df60 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/7289ddefc3ce4df99c329fe42c95df60 2024-04-14T11:16:13,411 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/7289ddefc3ce4df99c329fe42c95df60, entries=150, sequenceid=342, filesize=12.0 K 2024-04-14T11:16:13,412 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3035): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4dbfc9ae90980452da73dfe0910874fc in 983ms, sequenceid=342, compaction requested=true 2024-04-14T11:16:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2533): Flush status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=57 2024-04-14T11:16:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=57 2024-04-14T11:16:13,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=57, resume processing ppid=56 2024-04-14T11:16:13,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=57, ppid=56, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5990 sec 2024-04-14T11:16:13,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=56, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.6060 sec 2024-04-14T11:16:13,504 DEBUG [Thread-722 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4b6d84f7 to 127.0.0.1:60913 2024-04-14T11:16:13,504 DEBUG [Thread-722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:13,505 DEBUG [Thread-718 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7a25463a to 127.0.0.1:60913 2024-04-14T11:16:13,505 DEBUG [Thread-716 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x62a21f25 to 127.0.0.1:60913 2024-04-14T11:16:13,505 DEBUG [Thread-718 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:13,505 DEBUG [Thread-716 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:13,508 DEBUG [Thread-720 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7e2c3dae to 127.0.0.1:60913 2024-04-14T11:16:13,508 DEBUG [Thread-720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:13,733 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:13,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-04-14T11:16:13,733 DEBUG [Thread-707 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x36e1cef1 to 127.0.0.1:60913 2024-04-14T11:16:13,733 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:13,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414a5b31524329741d586c2b1ec1eb69600_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093372570/Put/seqid=0 2024-04-14T11:16:13,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=56 2024-04-14T11:16:13,921 INFO [Thread-715 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 56 completed 2024-04-14T11:16:14,153 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414a5b31524329741d586c2b1ec1eb69600_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414a5b31524329741d586c2b1ec1eb69600_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:14,155 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/631abcbc0e334756ad992a98dd8b69fc, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:14,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/631abcbc0e334756ad992a98dd8b69fc is 175, key is test_row_0/A:col10/1713093372570/Put/seqid=0 2024-04-14T11:16:14,167 DEBUG [Thread-705 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6365e97d to 127.0.0.1:60913 2024-04-14T11:16:14,167 DEBUG [Thread-705 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:14,564 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/631abcbc0e334756ad992a98dd8b69fc 2024-04-14T11:16:14,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/dd36e49a115b4e2e994b8a5a68bdaaca is 50, key is test_row_0/B:col10/1713093372570/Put/seqid=0 2024-04-14T11:16:14,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/dd36e49a115b4e2e994b8a5a68bdaaca 2024-04-14T11:16:14,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/4726d0614ead4f02808e5a09a0b8a9c8 is 50, key is test_row_0/C:col10/1713093372570/Put/seqid=0 2024-04-14T11:16:15,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/4726d0614ead4f02808e5a09a0b8a9c8 2024-04-14T11:16:15,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/631abcbc0e334756ad992a98dd8b69fc as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/631abcbc0e334756ad992a98dd8b69fc 2024-04-14T11:16:15,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/631abcbc0e334756ad992a98dd8b69fc, entries=150, sequenceid=368, filesize=30.5 K 2024-04-14T11:16:15,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/dd36e49a115b4e2e994b8a5a68bdaaca as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/dd36e49a115b4e2e994b8a5a68bdaaca 2024-04-14T11:16:15,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/dd36e49a115b4e2e994b8a5a68bdaaca, entries=150, sequenceid=368, filesize=12.0 K 2024-04-14T11:16:15,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/4726d0614ead4f02808e5a09a0b8a9c8 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/4726d0614ead4f02808e5a09a0b8a9c8 2024-04-14T11:16:15,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/4726d0614ead4f02808e5a09a0b8a9c8, entries=150, sequenceid=368, filesize=12.0 K 2024-04-14T11:16:15,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=6.71 KB/6870 for 4dbfc9ae90980452da73dfe0910874fc in 1683ms, sequenceid=368, compaction requested=true 2024-04-14T11:16:15,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:15,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:15,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:15,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:15,416 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:15,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:15,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4dbfc9ae90980452da73dfe0910874fc:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:15,417 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:15,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:15,418 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125702 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:15,418 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/A is initiating minor compaction (all files) 2024-04-14T11:16:15,418 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/A in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:15,418 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:15,418 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3198069c742c4e2391999f5558496cbf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/ff6bd2158b89473caa5763286b1c0dee, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3627b7f9726d48d6b8ce5e824fdc6758, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/631abcbc0e334756ad992a98dd8b69fc] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=122.8 K 2024-04-14T11:16:15,418 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/B is initiating minor compaction (all files) 2024-04-14T11:16:15,418 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:15,418 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3198069c742c4e2391999f5558496cbf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/ff6bd2158b89473caa5763286b1c0dee, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3627b7f9726d48d6b8ce5e824fdc6758, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/631abcbc0e334756ad992a98dd8b69fc] 2024-04-14T11:16:15,419 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/B in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:15,419 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/2c87253d19234b1db59b7fe70128f550, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/569c3aa2fd99439bb5faa4131c00ba5e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/ae476eedf9e44be99a2a2212b7b9815d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/dd36e49a115b4e2e994b8a5a68bdaaca] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=48.7 K 2024-04-14T11:16:15,419 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3198069c742c4e2391999f5558496cbf, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1713093368824 2024-04-14T11:16:15,419 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 2c87253d19234b1db59b7fe70128f550, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1713093368824 2024-04-14T11:16:15,419 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting ff6bd2158b89473caa5763286b1c0dee, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1713093370000 2024-04-14T11:16:15,420 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 569c3aa2fd99439bb5faa4131c00ba5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1713093370000 2024-04-14T11:16:15,420 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3627b7f9726d48d6b8ce5e824fdc6758, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1713093371878 2024-04-14T11:16:15,420 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting ae476eedf9e44be99a2a2212b7b9815d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1713093371878 2024-04-14T11:16:15,420 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 631abcbc0e334756ad992a98dd8b69fc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1713093372570 2024-04-14T11:16:15,420 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting dd36e49a115b4e2e994b8a5a68bdaaca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1713093372570 2024-04-14T11:16:15,433 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:15,434 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#B#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:15,435 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/05f6728b39f545d59ca9d741122dfb02 is 50, key is test_row_0/B:col10/1713093372570/Put/seqid=0 2024-04-14T11:16:15,435 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202404145c948b1ded57445bade00829e844c45d_4dbfc9ae90980452da73dfe0910874fc store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:15,462 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202404145c948b1ded57445bade00829e844c45d_4dbfc9ae90980452da73dfe0910874fc, store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:15,463 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404145c948b1ded57445bade00829e844c45d_4dbfc9ae90980452da73dfe0910874fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:15,851 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/05f6728b39f545d59ca9d741122dfb02 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/05f6728b39f545d59ca9d741122dfb02 2024-04-14T11:16:15,857 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/B of 4dbfc9ae90980452da73dfe0910874fc into 05f6728b39f545d59ca9d741122dfb02(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:15,857 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:15,857 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/B, priority=12, startTime=1713093375416; duration=0sec 2024-04-14T11:16:15,857 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:15,857 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:B 2024-04-14T11:16:15,857 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:15,859 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:15,859 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4dbfc9ae90980452da73dfe0910874fc/C is initiating minor compaction (all files) 2024-04-14T11:16:15,859 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4dbfc9ae90980452da73dfe0910874fc/C in TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:15,859 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/c44d49836f01402f8c1aa0c747482316, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/a34df326bc894cc28948252eb990f832, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/7289ddefc3ce4df99c329fe42c95df60, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/4726d0614ead4f02808e5a09a0b8a9c8] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp, totalSize=48.7 K 2024-04-14T11:16:15,860 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting c44d49836f01402f8c1aa0c747482316, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1713093368824 2024-04-14T11:16:15,860 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting a34df326bc894cc28948252eb990f832, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1713093370000 2024-04-14T11:16:15,861 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 7289ddefc3ce4df99c329fe42c95df60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1713093371878 2024-04-14T11:16:15,861 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 4726d0614ead4f02808e5a09a0b8a9c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1713093372570 2024-04-14T11:16:15,870 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#A#compaction#209 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:15,870 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/6414b45bcd84400fbb62cbe4542fef68 is 175, key is test_row_0/A:col10/1713093372570/Put/seqid=0 2024-04-14T11:16:15,871 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4dbfc9ae90980452da73dfe0910874fc#C#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:15,871 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/0cb5b3dff05c45dd8a0d6b541d640b45 is 50, key is test_row_0/C:col10/1713093372570/Put/seqid=0 2024-04-14T11:16:16,283 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/0cb5b3dff05c45dd8a0d6b541d640b45 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/0cb5b3dff05c45dd8a0d6b541d640b45 2024-04-14T11:16:16,283 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/6414b45bcd84400fbb62cbe4542fef68 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/6414b45bcd84400fbb62cbe4542fef68 2024-04-14T11:16:16,291 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/C of 4dbfc9ae90980452da73dfe0910874fc into 0cb5b3dff05c45dd8a0d6b541d640b45(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:16,291 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:16,291 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/C, priority=12, startTime=1713093375417; duration=0sec 2024-04-14T11:16:16,291 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:16,291 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:C 2024-04-14T11:16:16,291 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 4dbfc9ae90980452da73dfe0910874fc/A of 4dbfc9ae90980452da73dfe0910874fc into 6414b45bcd84400fbb62cbe4542fef68(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:16,291 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:16,292 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc., storeName=4dbfc9ae90980452da73dfe0910874fc/A, priority=12, startTime=1713093375416; duration=0sec 2024-04-14T11:16:16,292 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:16,292 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4dbfc9ae90980452da73dfe0910874fc:A 2024-04-14T11:16:16,962 DEBUG [Thread-713 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0fb426c5 to 127.0.0.1:60913 2024-04-14T11:16:16,962 DEBUG [Thread-713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:16,968 DEBUG [Thread-709 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x579b618d to 127.0.0.1:60913 2024-04-14T11:16:16,968 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:16,973 DEBUG [Thread-711 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1ec395f1 to 127.0.0.1:60913 2024-04-14T11:16:16,973 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:16,974 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-04-14T11:16:16,974 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-04-14T11:16:16,974 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-04-14T11:16:16,974 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-04-14T11:16:16,974 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-04-14T11:16:16,974 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-04-14T11:16:16,974 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-04-14T11:16:16,974 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(398): read 6706 2024-04-14T11:16:16,974 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(398): read 6695 2024-04-14T11:16:16,974 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-04-14T11:16:16,974 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(402): scanned 3062 2024-04-14T11:16:16,974 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(403): verified 9185 rows 2024-04-14T11:16:16,974 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(402): scanned 3078 2024-04-14T11:16:16,975 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(403): verified 9233 rows 2024-04-14T11:16:16,975 INFO [Listener at localhost/34025 {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-04-14T11:16:16,975 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x752ce290 to 127.0.0.1:60913 2024-04-14T11:16:16,975 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:16,977 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$18(968): Started disable of TestAcidGuarantees 2024-04-14T11:16:16,978 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$13(2744): Client=jenkins//172.31.2.20 disable TestAcidGuarantees 2024-04-14T11:16:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-04-14T11:16:16,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=58 2024-04-14T11:16:16,986 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093376986"}]},"ts":"1713093376986"} 2024-04-14T11:16:16,987 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-04-14T11:16:16,993 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set TestAcidGuarantees to state=DISABLING 2024-04-14T11:16:16,994 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=4dbfc9ae90980452da73dfe0910874fc, UNASSIGN}] 2024-04-14T11:16:16,995 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(729): Took xlock for pid=59, ppid=58, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=4dbfc9ae90980452da73dfe0910874fc, UNASSIGN 2024-04-14T11:16:16,996 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=59 updating hbase:meta row=4dbfc9ae90980452da73dfe0910874fc, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:16,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE; CloseRegionProcedure 4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:16:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=58 2024-04-14T11:16:17,148 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:17,149 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] handler.UnassignRegionHandler(119): Close 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:17,149 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] regionserver.HRegion(1676): Closing 4dbfc9ae90980452da73dfe0910874fc, disabling compactions & flushes 2024-04-14T11:16:17,149 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] regionserver.HRegion(1698): Closing region TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:17,149 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] regionserver.HRegion(1719): Waiting without time limit for close lock on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:17,149 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] regionserver.HRegion(1786): Acquired close lock on TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. after waiting 0 ms 2024-04-14T11:16:17,149 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] regionserver.HRegion(1796): Updates disabled for region TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:17,149 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] regionserver.HRegion(2832): Flushing 4dbfc9ae90980452da73dfe0910874fc 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-04-14T11:16:17,156 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414250212998278445e93622b88fbbde19b_4dbfc9ae90980452da73dfe0910874fc is 50, key is test_row_0/A:col10/1713093376972/Put/seqid=0 2024-04-14T11:16:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=58 2024-04-14T11:16:17,573 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414250212998278445e93622b88fbbde19b_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414250212998278445e93622b88fbbde19b_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:17,574 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/477fdd24eb9641c59609b40d51f34912, store: [table=TestAcidGuarantees family=A region=4dbfc9ae90980452da73dfe0910874fc] 2024-04-14T11:16:17,575 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/477fdd24eb9641c59609b40d51f34912 is 175, key is test_row_0/A:col10/1713093376972/Put/seqid=0 2024-04-14T11:16:17,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=58 2024-04-14T11:16:17,981 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=378, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/477fdd24eb9641c59609b40d51f34912 2024-04-14T11:16:17,989 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/4d08f32a6b5f4f799dced24c1c55be52 is 50, key is test_row_0/B:col10/1713093376972/Put/seqid=0 2024-04-14T11:16:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=58 2024-04-14T11:16:18,395 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/4d08f32a6b5f4f799dced24c1c55be52 2024-04-14T11:16:18,403 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/7a15a5271de948458e1ecd81e0589bdc is 50, key is test_row_0/C:col10/1713093376972/Put/seqid=0 2024-04-14T11:16:18,809 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/7a15a5271de948458e1ecd81e0589bdc 2024-04-14T11:16:18,817 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/A/477fdd24eb9641c59609b40d51f34912 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/477fdd24eb9641c59609b40d51f34912 2024-04-14T11:16:18,825 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/477fdd24eb9641c59609b40d51f34912, entries=100, sequenceid=378, filesize=22.0 K 2024-04-14T11:16:18,826 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/B/4d08f32a6b5f4f799dced24c1c55be52 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/4d08f32a6b5f4f799dced24c1c55be52 2024-04-14T11:16:18,833 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/4d08f32a6b5f4f799dced24c1c55be52, entries=100, sequenceid=378, filesize=9.6 K 2024-04-14T11:16:18,835 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/.tmp/C/7a15a5271de948458e1ecd81e0589bdc as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/7a15a5271de948458e1ecd81e0589bdc 2024-04-14T11:16:18,841 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/7a15a5271de948458e1ecd81e0589bdc, entries=100, sequenceid=378, filesize=9.6 K 2024-04-14T11:16:18,842 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] regionserver.HRegion(3035): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 4dbfc9ae90980452da73dfe0910874fc in 1693ms, sequenceid=378, compaction requested=false 2024-04-14T11:16:18,856 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/7b83cca8b5034935a236e192f1e69425, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/161ea00aa69c403bb5536e4aa672f77f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3a580da4b7cd48849047429c7a368189, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/60e2be28a35f452a8ecf3412cf25580f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/8e53f493c09841478dfb6f372a8ed804, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/caa965d687f44fbcb12adc7de15c00eb, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/bb0287312c084a89a4dd44abd71164eb, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3da9bd332f0c45dfbd947eea8882b329, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/b0fa2a4f72bb46ad97edbd5818e7cb0c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/c0d64ab28ab3427496728995bbc84661, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/c89a4cb1eddb4898ad269c3715d35c18, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/9f53502d158b4d5bb8d21df468f10984, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/6956369671494167b8d530ccdc1692ff, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/1e5fe4875c6c46c7933c5b59a61b15cb, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/4424a0c467b549cab7125d860e88904a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/38004244c086437881a76be0610f8025, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/cea404b346a44473832b9dec8cbf867f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/836e02e1be14442aa2ad16b7b7c1116f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/00284f8f6b1a46cb9dabff8239377673, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/036b2d15ec284c54b4141ffc881f3767, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/ed9dd625fd834e8bb9351e937414ed87, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/00f909093cb04963b4e744a5eb01442e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3198069c742c4e2391999f5558496cbf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/ff6bd2158b89473caa5763286b1c0dee, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3627b7f9726d48d6b8ce5e824fdc6758, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/631abcbc0e334756ad992a98dd8b69fc] to archive 2024-04-14T11:16:18,858 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:16:18,860 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/7b83cca8b5034935a236e192f1e69425 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/7b83cca8b5034935a236e192f1e69425 2024-04-14T11:16:18,862 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/161ea00aa69c403bb5536e4aa672f77f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/161ea00aa69c403bb5536e4aa672f77f 2024-04-14T11:16:18,864 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3a580da4b7cd48849047429c7a368189 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3a580da4b7cd48849047429c7a368189 2024-04-14T11:16:18,865 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/60e2be28a35f452a8ecf3412cf25580f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/60e2be28a35f452a8ecf3412cf25580f 2024-04-14T11:16:18,867 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/8e53f493c09841478dfb6f372a8ed804 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/8e53f493c09841478dfb6f372a8ed804 2024-04-14T11:16:18,868 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/caa965d687f44fbcb12adc7de15c00eb to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/caa965d687f44fbcb12adc7de15c00eb 2024-04-14T11:16:18,870 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/bb0287312c084a89a4dd44abd71164eb to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/bb0287312c084a89a4dd44abd71164eb 2024-04-14T11:16:18,871 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3da9bd332f0c45dfbd947eea8882b329 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3da9bd332f0c45dfbd947eea8882b329 2024-04-14T11:16:18,873 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/b0fa2a4f72bb46ad97edbd5818e7cb0c to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/b0fa2a4f72bb46ad97edbd5818e7cb0c 2024-04-14T11:16:18,874 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/c0d64ab28ab3427496728995bbc84661 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/c0d64ab28ab3427496728995bbc84661 2024-04-14T11:16:18,876 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/c89a4cb1eddb4898ad269c3715d35c18 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/c89a4cb1eddb4898ad269c3715d35c18 2024-04-14T11:16:18,878 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/9f53502d158b4d5bb8d21df468f10984 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/9f53502d158b4d5bb8d21df468f10984 2024-04-14T11:16:18,879 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/6956369671494167b8d530ccdc1692ff to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/6956369671494167b8d530ccdc1692ff 2024-04-14T11:16:18,880 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/1e5fe4875c6c46c7933c5b59a61b15cb to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/1e5fe4875c6c46c7933c5b59a61b15cb 2024-04-14T11:16:18,883 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/4424a0c467b549cab7125d860e88904a to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/4424a0c467b549cab7125d860e88904a 2024-04-14T11:16:18,884 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/38004244c086437881a76be0610f8025 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/38004244c086437881a76be0610f8025 2024-04-14T11:16:18,886 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/cea404b346a44473832b9dec8cbf867f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/cea404b346a44473832b9dec8cbf867f 2024-04-14T11:16:18,887 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/836e02e1be14442aa2ad16b7b7c1116f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/836e02e1be14442aa2ad16b7b7c1116f 2024-04-14T11:16:18,888 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/00284f8f6b1a46cb9dabff8239377673 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/00284f8f6b1a46cb9dabff8239377673 2024-04-14T11:16:18,890 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/036b2d15ec284c54b4141ffc881f3767 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/036b2d15ec284c54b4141ffc881f3767 2024-04-14T11:16:18,891 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/ed9dd625fd834e8bb9351e937414ed87 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/ed9dd625fd834e8bb9351e937414ed87 2024-04-14T11:16:18,893 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/00f909093cb04963b4e744a5eb01442e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/00f909093cb04963b4e744a5eb01442e 2024-04-14T11:16:18,894 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3198069c742c4e2391999f5558496cbf to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3198069c742c4e2391999f5558496cbf 2024-04-14T11:16:18,896 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/ff6bd2158b89473caa5763286b1c0dee to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/ff6bd2158b89473caa5763286b1c0dee 2024-04-14T11:16:18,897 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3627b7f9726d48d6b8ce5e824fdc6758 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/3627b7f9726d48d6b8ce5e824fdc6758 2024-04-14T11:16:18,899 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/631abcbc0e334756ad992a98dd8b69fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/631abcbc0e334756ad992a98dd8b69fc 2024-04-14T11:16:18,900 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/7bd6b94bcee143e88a481d3790046d86, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/27b75dd1902849d1a82a9a9b942583cc, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/1a4f42a668c743f6937f51dc72670dc0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/2d6f1f2e3c9f4066b536c92df6774af4, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/05296ede409a4dde891c0a2717c2b06a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/dee38c379cdb4c9a891bf6da8751e2c3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/d608047cdeb04fef831e0d0f770276db, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/13a26856f28c4e74aeebacdd1effc56f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/dffec1432ebb4a2bb64d5e053ceacec8, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/25a0cba62b164ee3a658c93516f58452, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/4bc27bcef64f4675abe0a657415df6e7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/1bff09597ce340a8a092e45f69d4f77d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/f872e92202734c7b8ad070c0c5c69ec6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/67e9a8cbc3ea49919845b4a12292f2ff, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/8b82ce200a454e00be95174a080d52d5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/c5b9d60bd46d4a88a246992798e995a1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/e0c6cdf10d17451daa981752c5bb2140, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/35d7fbbc30774db7809ca4d5df3846a2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/e8d4f73e01fc402b9484ea06092e6ba9, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/2a62a8ec48c04673b810bcc9f9fff216, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/bc71dfbfa442458783eb63730ff5c87c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/2c87253d19234b1db59b7fe70128f550, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/db253e82adc341a292f4df3dfaddfd61, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/569c3aa2fd99439bb5faa4131c00ba5e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/ae476eedf9e44be99a2a2212b7b9815d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/dd36e49a115b4e2e994b8a5a68bdaaca] to archive 2024-04-14T11:16:18,902 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:16:18,903 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/7bd6b94bcee143e88a481d3790046d86 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/7bd6b94bcee143e88a481d3790046d86 2024-04-14T11:16:18,905 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/27b75dd1902849d1a82a9a9b942583cc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/27b75dd1902849d1a82a9a9b942583cc 2024-04-14T11:16:18,906 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/1a4f42a668c743f6937f51dc72670dc0 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/1a4f42a668c743f6937f51dc72670dc0 2024-04-14T11:16:18,908 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/2d6f1f2e3c9f4066b536c92df6774af4 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/2d6f1f2e3c9f4066b536c92df6774af4 2024-04-14T11:16:18,909 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/05296ede409a4dde891c0a2717c2b06a to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/05296ede409a4dde891c0a2717c2b06a 2024-04-14T11:16:18,910 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/dee38c379cdb4c9a891bf6da8751e2c3 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/dee38c379cdb4c9a891bf6da8751e2c3 2024-04-14T11:16:18,912 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/d608047cdeb04fef831e0d0f770276db to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/d608047cdeb04fef831e0d0f770276db 2024-04-14T11:16:18,913 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/13a26856f28c4e74aeebacdd1effc56f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/13a26856f28c4e74aeebacdd1effc56f 2024-04-14T11:16:18,914 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/dffec1432ebb4a2bb64d5e053ceacec8 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/dffec1432ebb4a2bb64d5e053ceacec8 2024-04-14T11:16:18,916 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/25a0cba62b164ee3a658c93516f58452 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/25a0cba62b164ee3a658c93516f58452 2024-04-14T11:16:18,917 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/4bc27bcef64f4675abe0a657415df6e7 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/4bc27bcef64f4675abe0a657415df6e7 2024-04-14T11:16:18,918 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/1bff09597ce340a8a092e45f69d4f77d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/1bff09597ce340a8a092e45f69d4f77d 2024-04-14T11:16:18,920 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/f872e92202734c7b8ad070c0c5c69ec6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/f872e92202734c7b8ad070c0c5c69ec6 2024-04-14T11:16:18,921 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/67e9a8cbc3ea49919845b4a12292f2ff to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/67e9a8cbc3ea49919845b4a12292f2ff 2024-04-14T11:16:18,922 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/8b82ce200a454e00be95174a080d52d5 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/8b82ce200a454e00be95174a080d52d5 2024-04-14T11:16:18,924 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/c5b9d60bd46d4a88a246992798e995a1 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/c5b9d60bd46d4a88a246992798e995a1 2024-04-14T11:16:18,925 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/e0c6cdf10d17451daa981752c5bb2140 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/e0c6cdf10d17451daa981752c5bb2140 2024-04-14T11:16:18,926 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/35d7fbbc30774db7809ca4d5df3846a2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/35d7fbbc30774db7809ca4d5df3846a2 2024-04-14T11:16:18,927 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/e8d4f73e01fc402b9484ea06092e6ba9 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/e8d4f73e01fc402b9484ea06092e6ba9 2024-04-14T11:16:18,929 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/2a62a8ec48c04673b810bcc9f9fff216 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/2a62a8ec48c04673b810bcc9f9fff216 2024-04-14T11:16:18,930 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/bc71dfbfa442458783eb63730ff5c87c to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/bc71dfbfa442458783eb63730ff5c87c 2024-04-14T11:16:18,931 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/2c87253d19234b1db59b7fe70128f550 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/2c87253d19234b1db59b7fe70128f550 2024-04-14T11:16:18,933 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/db253e82adc341a292f4df3dfaddfd61 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/db253e82adc341a292f4df3dfaddfd61 2024-04-14T11:16:18,935 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/569c3aa2fd99439bb5faa4131c00ba5e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/569c3aa2fd99439bb5faa4131c00ba5e 2024-04-14T11:16:18,936 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/ae476eedf9e44be99a2a2212b7b9815d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/ae476eedf9e44be99a2a2212b7b9815d 2024-04-14T11:16:18,938 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/dd36e49a115b4e2e994b8a5a68bdaaca to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/dd36e49a115b4e2e994b8a5a68bdaaca 2024-04-14T11:16:18,939 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/d2eb2bc257de458482f60fadd297664d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/4514842d859b48c2a75624da615aea56, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/e913bc46e95c4807a237d89c497e3d72, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/4cacec1671fb4590b249acda23d6b1e2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/93439a0a8aad4e6cabca5b509be497ee, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/e737e028157d47c4a6990930b2519cb2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/ec40a0f30d7e4184b627a607fe0e4386, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/1cbf5b5d10074761b1d940372b3eb958, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/a36c5cd0f66b4d0d9af59549cc677d02, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/54d2d1972cfd4b9caf0f69c41439edf9, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/8a344fbb41714817b21dc12d0d17fb15, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/8889e23432f24182b68213b88a4f6f35, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/8b2a5cbbddcd4930be15c1fcef476e2d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/7c11c926ffe74ce990c14b2fb0751fc6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/5cff079c87a547fd92590a9da1ea3969, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/0879b9a7ebab4e3ba2d6e56470d6bd9b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/e997bc7721b049f3bf53048743d08e4b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/39efda50c4724f909e7c0e27eb968d5a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/5fd1f9ba17ca4449b9ec8ebc3719b160, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/93036490485145e8906f8f80ef098e42, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/bd7129104ef54e158e2622767dff61bd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/c44d49836f01402f8c1aa0c747482316, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/cde070c111144a6481d167c305c265c6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/a34df326bc894cc28948252eb990f832, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/7289ddefc3ce4df99c329fe42c95df60, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/4726d0614ead4f02808e5a09a0b8a9c8] to archive 2024-04-14T11:16:18,941 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:16:18,943 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/d2eb2bc257de458482f60fadd297664d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/d2eb2bc257de458482f60fadd297664d 2024-04-14T11:16:18,944 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/4514842d859b48c2a75624da615aea56 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/4514842d859b48c2a75624da615aea56 2024-04-14T11:16:18,946 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/e913bc46e95c4807a237d89c497e3d72 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/e913bc46e95c4807a237d89c497e3d72 2024-04-14T11:16:18,948 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/4cacec1671fb4590b249acda23d6b1e2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/4cacec1671fb4590b249acda23d6b1e2 2024-04-14T11:16:18,950 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/93439a0a8aad4e6cabca5b509be497ee to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/93439a0a8aad4e6cabca5b509be497ee 2024-04-14T11:16:18,952 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/e737e028157d47c4a6990930b2519cb2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/e737e028157d47c4a6990930b2519cb2 2024-04-14T11:16:18,953 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/ec40a0f30d7e4184b627a607fe0e4386 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/ec40a0f30d7e4184b627a607fe0e4386 2024-04-14T11:16:18,955 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/1cbf5b5d10074761b1d940372b3eb958 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/1cbf5b5d10074761b1d940372b3eb958 2024-04-14T11:16:18,957 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/a36c5cd0f66b4d0d9af59549cc677d02 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/a36c5cd0f66b4d0d9af59549cc677d02 2024-04-14T11:16:18,959 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/54d2d1972cfd4b9caf0f69c41439edf9 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/54d2d1972cfd4b9caf0f69c41439edf9 2024-04-14T11:16:18,960 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/8a344fbb41714817b21dc12d0d17fb15 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/8a344fbb41714817b21dc12d0d17fb15 2024-04-14T11:16:18,962 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/8889e23432f24182b68213b88a4f6f35 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/8889e23432f24182b68213b88a4f6f35 2024-04-14T11:16:18,963 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/8b2a5cbbddcd4930be15c1fcef476e2d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/8b2a5cbbddcd4930be15c1fcef476e2d 2024-04-14T11:16:18,964 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/7c11c926ffe74ce990c14b2fb0751fc6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/7c11c926ffe74ce990c14b2fb0751fc6 2024-04-14T11:16:18,966 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/5cff079c87a547fd92590a9da1ea3969 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/5cff079c87a547fd92590a9da1ea3969 2024-04-14T11:16:18,968 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/0879b9a7ebab4e3ba2d6e56470d6bd9b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/0879b9a7ebab4e3ba2d6e56470d6bd9b 2024-04-14T11:16:18,969 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/e997bc7721b049f3bf53048743d08e4b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/e997bc7721b049f3bf53048743d08e4b 2024-04-14T11:16:18,971 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/39efda50c4724f909e7c0e27eb968d5a to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/39efda50c4724f909e7c0e27eb968d5a 2024-04-14T11:16:18,973 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/5fd1f9ba17ca4449b9ec8ebc3719b160 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/5fd1f9ba17ca4449b9ec8ebc3719b160 2024-04-14T11:16:18,975 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/93036490485145e8906f8f80ef098e42 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/93036490485145e8906f8f80ef098e42 2024-04-14T11:16:18,976 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/bd7129104ef54e158e2622767dff61bd to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/bd7129104ef54e158e2622767dff61bd 2024-04-14T11:16:18,978 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/c44d49836f01402f8c1aa0c747482316 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/c44d49836f01402f8c1aa0c747482316 2024-04-14T11:16:18,979 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/cde070c111144a6481d167c305c265c6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/cde070c111144a6481d167c305c265c6 2024-04-14T11:16:18,980 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/a34df326bc894cc28948252eb990f832 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/a34df326bc894cc28948252eb990f832 2024-04-14T11:16:18,981 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/7289ddefc3ce4df99c329fe42c95df60 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/7289ddefc3ce4df99c329fe42c95df60 2024-04-14T11:16:18,983 DEBUG [StoreCloser-TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/4726d0614ead4f02808e5a09a0b8a9c8 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/4726d0614ead4f02808e5a09a0b8a9c8 2024-04-14T11:16:18,993 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/recovered.edits/381.seqid, newMaxSeqId=381, maxSeqId=4 2024-04-14T11:16:18,995 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] regionserver.HRegion(1917): Closed TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc. 2024-04-14T11:16:18,995 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] regionserver.HRegion(1630): Region close journal for 4dbfc9ae90980452da73dfe0910874fc: 2024-04-14T11:16:18,998 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=60}] handler.UnassignRegionHandler(163): Closed 4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:18,998 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=59 updating hbase:meta row=4dbfc9ae90980452da73dfe0910874fc, regionState=CLOSED 2024-04-14T11:16:19,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=60, resume processing ppid=59 2024-04-14T11:16:19,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=60, ppid=59, state=SUCCESS; CloseRegionProcedure 4dbfc9ae90980452da73dfe0910874fc, server=jenkins-hbase6.apache.org,33239,1713093321027 in 2.0020 sec 2024-04-14T11:16:19,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=59, resume processing ppid=58 2024-04-14T11:16:19,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=59, ppid=58, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=4dbfc9ae90980452da73dfe0910874fc, UNASSIGN in 2.0070 sec 2024-04-14T11:16:19,006 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093379005"}]},"ts":"1713093379005"} 2024-04-14T11:16:19,007 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-04-14T11:16:19,011 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set TestAcidGuarantees to state=DISABLED 2024-04-14T11:16:19,013 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=58, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 2.0330 sec 2024-04-14T11:16:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=58 2024-04-14T11:16:19,090 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$TableFuture(3746): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 58 completed 2024-04-14T11:16:19,090 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$5(2494): Client=jenkins//172.31.2.20 delete TestAcidGuarantees 2024-04-14T11:16:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=61, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:16:19,093 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=61, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:16:19,094 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=61, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:16:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=61 2024-04-14T11:16:19,097 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,100 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A, FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B, FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C, FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/recovered.edits] 2024-04-14T11:16:19,104 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/477fdd24eb9641c59609b40d51f34912 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/477fdd24eb9641c59609b40d51f34912 2024-04-14T11:16:19,106 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/6414b45bcd84400fbb62cbe4542fef68 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/A/6414b45bcd84400fbb62cbe4542fef68 2024-04-14T11:16:19,109 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/05f6728b39f545d59ca9d741122dfb02 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/05f6728b39f545d59ca9d741122dfb02 2024-04-14T11:16:19,111 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/4d08f32a6b5f4f799dced24c1c55be52 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/B/4d08f32a6b5f4f799dced24c1c55be52 2024-04-14T11:16:19,114 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/0cb5b3dff05c45dd8a0d6b541d640b45 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/0cb5b3dff05c45dd8a0d6b541d640b45 2024-04-14T11:16:19,116 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/7a15a5271de948458e1ecd81e0589bdc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/C/7a15a5271de948458e1ecd81e0589bdc 2024-04-14T11:16:19,120 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/recovered.edits/381.seqid to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc/recovered.edits/381.seqid 2024-04-14T11:16:19,121 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,121 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived TestAcidGuarantees regions 2024-04-14T11:16:19,122 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-04-14T11:16:19,123 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-04-14T11:16:19,127 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414075ab0fcbf30489aae14c989edceedba_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414075ab0fcbf30489aae14c989edceedba_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,128 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404140797e940ec8b49da94b90a6b06fde953_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404140797e940ec8b49da94b90a6b06fde953_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,130 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404140a9921bec4dc4c3e945393e3660144a8_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404140a9921bec4dc4c3e945393e3660144a8_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,131 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404140ab4e8f2b1c1407baf5579b70c40338d_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404140ab4e8f2b1c1407baf5579b70c40338d_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,132 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404140b1061a98c164662a322d9ee0f69cee2_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404140b1061a98c164662a322d9ee0f69cee2_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,134 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414148d27e835ae4711a420a500bce47b56_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414148d27e835ae4711a420a500bce47b56_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,136 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041424fc14d0c65f4415a6707f7b9b866d20_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041424fc14d0c65f4415a6707f7b9b866d20_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,137 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414250212998278445e93622b88fbbde19b_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414250212998278445e93622b88fbbde19b_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,139 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414338ba9b6eb0e4a7695f3f6f5e0229c7b_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414338ba9b6eb0e4a7695f3f6f5e0229c7b_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,141 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414524ff5d0499f4ee8b2ef4a61ef4ca48f_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414524ff5d0499f4ee8b2ef4a61ef4ca48f_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,143 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404145799a70ab8b14675ac0cb8a85f4c0357_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404145799a70ab8b14675ac0cb8a85f4c0357_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,145 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041485184cc896af4505b0b2add39fd3a3f4_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041485184cc896af4505b0b2add39fd3a3f4_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,148 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041489fb0a40a38e4d08811af8d46f3d3896_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041489fb0a40a38e4d08811af8d46f3d3896_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,155 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414a5b31524329741d586c2b1ec1eb69600_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414a5b31524329741d586c2b1ec1eb69600_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,157 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414b40e19a0fbd44102b787ca8ad44ec342_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414b40e19a0fbd44102b787ca8ad44ec342_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,159 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414c013e2087f864f9b97ad474fe8f44821_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414c013e2087f864f9b97ad474fe8f44821_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,160 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414db502b3d83dd4fb28b4323a28cc981af_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414db502b3d83dd4fb28b4323a28cc981af_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,162 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414dd3fc829690443d89390a3b84695f141_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414dd3fc829690443d89390a3b84695f141_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,164 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414dd411552a87643ecaebaad3fe572d2c2_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414dd411552a87643ecaebaad3fe572d2c2_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,166 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414f56af5548c324c6abc45e3427f6f31eb_4dbfc9ae90980452da73dfe0910874fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414f56af5548c324c6abc45e3427f6f31eb_4dbfc9ae90980452da73dfe0910874fc 2024-04-14T11:16:19,167 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-04-14T11:16:19,170 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=61, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:16:19,174 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-04-14T11:16:19,176 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'TestAcidGuarantees' descriptor. 2024-04-14T11:16:19,177 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=61, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:16:19,177 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'TestAcidGuarantees' from region states. 2024-04-14T11:16:19,178 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1713093379177"}]},"ts":"9223372036854775807"} 2024-04-14T11:16:19,180 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-04-14T11:16:19,180 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 4dbfc9ae90980452da73dfe0910874fc, NAME => 'TestAcidGuarantees,,1713093350293.4dbfc9ae90980452da73dfe0910874fc.', STARTKEY => '', ENDKEY => ''}] 2024-04-14T11:16:19,180 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'TestAcidGuarantees' as deleted. 2024-04-14T11:16:19,180 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1713093379180"}]},"ts":"9223372036854775807"} 2024-04-14T11:16:19,182 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-04-14T11:16:19,185 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=61, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:16:19,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=61, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 95 msec 2024-04-14T11:16:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=61 2024-04-14T11:16:19,196 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$TableFuture(3746): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 61 completed 2024-04-14T11:16:19,212 INFO [Listener at localhost/34025 {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=224 (was 223) - Thread LEAK? -, OpenFileDescriptor=531 (was 525) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=459 (was 381) - SystemLoadAverage LEAK? -, ProcessCount=168 (was 168), AvailableMemoryMB=2408 (was 2438) 2024-04-14T11:16:19,226 INFO [Listener at localhost/34025 {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=224, OpenFileDescriptor=531, MaxFileDescriptor=60000, SystemLoadAverage=459, ProcessCount=168, AvailableMemoryMB=2408 2024-04-14T11:16:19,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-04-14T11:16:19,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$4(2378): Client=jenkins//172.31.2.20 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-04-14T11:16:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=62, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-04-14T11:16:19,231 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(84): pid=62, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-04-14T11:16:19,231 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:16:19,232 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(84): pid=62, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-04-14T11:16:19,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 62 2024-04-14T11:16:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2024-04-14T11:16:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2024-04-14T11:16:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2024-04-14T11:16:19,642 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7088): creating {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a 2024-04-14T11:16:19,648 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(891): Instantiated TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:16:19,649 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Closing c0901aa47578fba1a7f58734a4a0e3b2, disabling compactions & flushes 2024-04-14T11:16:19,649 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1698): Closing region TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:19,649 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1719): Waiting without time limit for close lock on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:19,649 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1786): Acquired close lock on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. after waiting 0 ms 2024-04-14T11:16:19,649 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1796): Updates disabled for region TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:19,649 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1917): Closed TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:19,649 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1630): Region close journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:19,650 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(84): pid=62, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-04-14T11:16:19,650 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1713093379650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1713093379650"}]},"ts":"1713093379650"} 2024-04-14T11:16:19,651 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-04-14T11:16:19,652 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(84): pid=62, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-04-14T11:16:19,652 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093379652"}]},"ts":"1713093379652"} 2024-04-14T11:16:19,653 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-04-14T11:16:19,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0901aa47578fba1a7f58734a4a0e3b2, ASSIGN}] 2024-04-14T11:16:19,660 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(729): Took xlock for pid=63, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0901aa47578fba1a7f58734a4a0e3b2, ASSIGN 2024-04-14T11:16:19,661 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=63, ppid=62, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0901aa47578fba1a7f58734a4a0e3b2, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,33239,1713093321027; forceNewPlan=false, retain=false 2024-04-14T11:16:19,812 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=63 updating hbase:meta row=c0901aa47578fba1a7f58734a4a0e3b2, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:19,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE; OpenRegionProcedure c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:16:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2024-04-14T11:16:19,966 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:19,970 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:19,970 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(7267): Opening region: {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} 2024-04-14T11:16:19,970 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:19,970 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(891): Instantiated TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:16:19,970 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(7309): checking encryption for c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:19,970 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(7312): checking classloading for c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:19,972 INFO [StoreOpener-c0901aa47578fba1a7f58734a4a0e3b2-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:19,973 INFO [StoreOpener-c0901aa47578fba1a7f58734a4a0e3b2-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0901aa47578fba1a7f58734a4a0e3b2 columnFamilyName A 2024-04-14T11:16:19,973 DEBUG [StoreOpener-c0901aa47578fba1a7f58734a4a0e3b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:16:19,974 INFO [StoreOpener-c0901aa47578fba1a7f58734a4a0e3b2-1 {}] regionserver.HStore(324): Store=c0901aa47578fba1a7f58734a4a0e3b2/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:16:19,974 INFO [StoreOpener-c0901aa47578fba1a7f58734a4a0e3b2-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:19,975 INFO [StoreOpener-c0901aa47578fba1a7f58734a4a0e3b2-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0901aa47578fba1a7f58734a4a0e3b2 columnFamilyName B 2024-04-14T11:16:19,975 DEBUG [StoreOpener-c0901aa47578fba1a7f58734a4a0e3b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:16:19,975 INFO [StoreOpener-c0901aa47578fba1a7f58734a4a0e3b2-1 {}] regionserver.HStore(324): Store=c0901aa47578fba1a7f58734a4a0e3b2/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:16:19,975 INFO [StoreOpener-c0901aa47578fba1a7f58734a4a0e3b2-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:19,977 INFO [StoreOpener-c0901aa47578fba1a7f58734a4a0e3b2-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0901aa47578fba1a7f58734a4a0e3b2 columnFamilyName C 2024-04-14T11:16:19,977 DEBUG [StoreOpener-c0901aa47578fba1a7f58734a4a0e3b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:16:19,977 INFO [StoreOpener-c0901aa47578fba1a7f58734a4a0e3b2-1 {}] regionserver.HStore(324): Store=c0901aa47578fba1a7f58734a4a0e3b2/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:16:19,978 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:19,979 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:19,980 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-04-14T11:16:19,982 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(1082): writing seq id for c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:19,984 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-04-14T11:16:19,984 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(1099): Opened c0901aa47578fba1a7f58734a4a0e3b2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70384509, jitterRate=0.048810914158821106}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-04-14T11:16:19,985 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegion(998): Region open journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:19,985 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegionServer(2593): Post open deploy tasks for TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., pid=64, masterSystemTime=1713093379965 2024-04-14T11:16:19,987 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] regionserver.HRegionServer(2620): Finished post open deploy task for TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:19,987 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=64}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:19,987 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=63 updating hbase:meta row=c0901aa47578fba1a7f58734a4a0e3b2, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:19,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=64, resume processing ppid=63 2024-04-14T11:16:19,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=64, ppid=63, state=SUCCESS; OpenRegionProcedure c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 in 174 msec 2024-04-14T11:16:19,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=63, resume processing ppid=62 2024-04-14T11:16:19,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=63, ppid=62, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0901aa47578fba1a7f58734a4a0e3b2, ASSIGN in 331 msec 2024-04-14T11:16:19,992 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(84): pid=62, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-04-14T11:16:19,992 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093379992"}]},"ts":"1713093379992"} 2024-04-14T11:16:19,993 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-04-14T11:16:19,996 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(84): pid=62, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-04-14T11:16:19,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=62, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 768 msec 2024-04-14T11:16:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2024-04-14T11:16:20,338 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 62 completed 2024-04-14T11:16:20,340 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2e20f4a4 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:20,346 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@584434fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:20,348 DEBUG [Listener at localhost/34025 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:20,350 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:58112, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:20,352 DEBUG [Listener at localhost/34025 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-04-14T11:16:20,353 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:40072, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-04-14T11:16:20,356 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5e76c41d to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:20,361 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77da8ea3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:20,363 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0dd7f4f1 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:20,371 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65e99f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:20,372 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4dd5fe20 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:20,380 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f97fb17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:20,381 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3d752bcd to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:20,390 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fb3f6bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:20,391 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2ceda901 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:20,402 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3852d412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:20,403 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x41bcfd04 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:20,417 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fe04365, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:20,418 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x76a586b1 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:20,425 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@505701bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:20,426 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x349e56ad to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:20,433 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e54de81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:20,435 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x374b3f29 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:20,440 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71263c53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:20,441 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0cdd2dc7 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:20,446 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6281e902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:20,454 DEBUG [hconnection-0x3095978f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:20,455 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:20,457 DEBUG [hconnection-0x6d694e3-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:20,458 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:58122, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:20,463 DEBUG [hconnection-0x119f4ec5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:20,463 DEBUG [hconnection-0x11217a77-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:20,465 DEBUG [hconnection-0x7cb02f93-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:20,466 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:58136, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:20,473 DEBUG [hconnection-0x20bdaaac-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:20,473 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:58128, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:20,479 DEBUG [hconnection-0x71d539be-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:20,479 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:58144, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:20,479 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:58116, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=65, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=65, table=TestAcidGuarantees 2024-04-14T11:16:20,480 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:58132, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=65 2024-04-14T11:16:20,487 DEBUG [hconnection-0xef864bd-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:20,487 DEBUG [hconnection-0x563d8558-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:20,487 DEBUG [hconnection-0x166d6265-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:20,489 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:58152, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:20,490 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:58166, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:20,491 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:58174, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:20,492 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:58188, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:20,492 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=65, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=65, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:20,495 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=65, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=65, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:20,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:20,503 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:20,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-04-14T11:16:20,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/b5f0e269708f4170b0266c2ee661c20c is 50, key is test_row_0/A:col10/1713093380499/Put/seqid=0 2024-04-14T11:16:20,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093440536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093440538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093440538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093440538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093440539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/b5f0e269708f4170b0266c2ee661c20c 2024-04-14T11:16:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=65 2024-04-14T11:16:20,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/abd04d31efdb4bb98d5dda74c65288b0 is 50, key is test_row_0/B:col10/1713093380499/Put/seqid=0 2024-04-14T11:16:20,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/abd04d31efdb4bb98d5dda74c65288b0 2024-04-14T11:16:20,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093440640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093440643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093440643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093440644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093440644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,649 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=66 2024-04-14T11:16:20,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:20,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:20,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:20,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] handler.RSProcedureHandler(53): pid=66 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:20,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=66 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:20,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=66 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:20,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/33f990f89a6044108dc8f3e9774fa980 is 50, key is test_row_0/C:col10/1713093380499/Put/seqid=0 2024-04-14T11:16:20,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/33f990f89a6044108dc8f3e9774fa980 2024-04-14T11:16:20,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/b5f0e269708f4170b0266c2ee661c20c as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/b5f0e269708f4170b0266c2ee661c20c 2024-04-14T11:16:20,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/b5f0e269708f4170b0266c2ee661c20c, entries=150, sequenceid=13, filesize=11.7 K 2024-04-14T11:16:20,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/abd04d31efdb4bb98d5dda74c65288b0 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/abd04d31efdb4bb98d5dda74c65288b0 2024-04-14T11:16:20,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/abd04d31efdb4bb98d5dda74c65288b0, entries=150, sequenceid=13, filesize=11.7 K 2024-04-14T11:16:20,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/33f990f89a6044108dc8f3e9774fa980 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/33f990f89a6044108dc8f3e9774fa980 2024-04-14T11:16:20,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/33f990f89a6044108dc8f3e9774fa980, entries=150, sequenceid=13, filesize=11.7 K 2024-04-14T11:16:20,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c0901aa47578fba1a7f58734a4a0e3b2 in 209ms, sequenceid=13, compaction requested=false 2024-04-14T11:16:20,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:20,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=65 2024-04-14T11:16:20,802 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=66 2024-04-14T11:16:20,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:20,805 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-04-14T11:16:20,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/2dc3830742044f47b3f5ab267b3a1a41 is 50, key is test_row_0/A:col10/1713093380536/Put/seqid=0 2024-04-14T11:16:20,823 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/2dc3830742044f47b3f5ab267b3a1a41 2024-04-14T11:16:20,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/788dc4c85ec4450fabcb862c7411afc4 is 50, key is test_row_0/B:col10/1713093380536/Put/seqid=0 2024-04-14T11:16:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:20,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:20,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093440857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093440860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093440861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093440866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093440867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093440967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093440968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093440968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093440970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:20,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093440974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=65 2024-04-14T11:16:21,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093441171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093441174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093441176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093441177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093441178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,277 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/788dc4c85ec4450fabcb862c7411afc4 2024-04-14T11:16:21,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/23c9766ea79844c9a851602294bc32e6 is 50, key is test_row_0/C:col10/1713093380536/Put/seqid=0 2024-04-14T11:16:21,319 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/23c9766ea79844c9a851602294bc32e6 2024-04-14T11:16:21,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/2dc3830742044f47b3f5ab267b3a1a41 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2dc3830742044f47b3f5ab267b3a1a41 2024-04-14T11:16:21,338 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2dc3830742044f47b3f5ab267b3a1a41, entries=150, sequenceid=37, filesize=11.7 K 2024-04-14T11:16:21,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/788dc4c85ec4450fabcb862c7411afc4 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/788dc4c85ec4450fabcb862c7411afc4 2024-04-14T11:16:21,346 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/788dc4c85ec4450fabcb862c7411afc4, entries=150, sequenceid=37, filesize=11.7 K 2024-04-14T11:16:21,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/23c9766ea79844c9a851602294bc32e6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/23c9766ea79844c9a851602294bc32e6 2024-04-14T11:16:21,352 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/23c9766ea79844c9a851602294bc32e6, entries=150, sequenceid=37, filesize=11.7 K 2024-04-14T11:16:21,353 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegion(3035): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for c0901aa47578fba1a7f58734a4a0e3b2 in 548ms, sequenceid=37, compaction requested=false 2024-04-14T11:16:21,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:21,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:21,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=66}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=66 2024-04-14T11:16:21,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=66 2024-04-14T11:16:21,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=66, resume processing ppid=65 2024-04-14T11:16:21,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=66, ppid=65, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 860 msec 2024-04-14T11:16:21,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=65, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=65, table=TestAcidGuarantees in 905 msec 2024-04-14T11:16:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:21,480 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:21,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-04-14T11:16:21,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/2b71dbcc28df4fd9a425065b9bdee307 is 50, key is test_row_0/A:col10/1713093381480/Put/seqid=0 2024-04-14T11:16:21,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093441538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093441541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093441541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093441541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093441543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=65 2024-04-14T11:16:21,602 INFO [Thread-1198 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 65 completed 2024-04-14T11:16:21,604 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:21,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=67, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=67, table=TestAcidGuarantees 2024-04-14T11:16:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=67 2024-04-14T11:16:21,609 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=67, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=67, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:21,610 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=67, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=67, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:21,610 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:21,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093441645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093441645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093441646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093441646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093441650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=67 2024-04-14T11:16:21,761 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=68 2024-04-14T11:16:21,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:21,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:21,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:21,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=68}] handler.RSProcedureHandler(53): pid=68 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:21,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=68 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=68 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:21,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093441848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093441848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093441850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093441851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093441853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=67 2024-04-14T11:16:21,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/2b71dbcc28df4fd9a425065b9bdee307 2024-04-14T11:16:21,915 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:21,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=68 2024-04-14T11:16:21,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:21,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:21,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:21,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=68}] handler.RSProcedureHandler(53): pid=68 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:21,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=68 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=68 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:21,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/9a84a28639f14d8c9f09dcd386c3226c is 50, key is test_row_0/B:col10/1713093381480/Put/seqid=0 2024-04-14T11:16:22,068 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:22,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=68 2024-04-14T11:16:22,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:22,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:22,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:22,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] handler.RSProcedureHandler(53): pid=68 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:22,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=68 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=68 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:22,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093442152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:22,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093442154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:22,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093442154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:22,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093442156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:22,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093442156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=67 2024-04-14T11:16:22,221 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:22,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=68 2024-04-14T11:16:22,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:22,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:22,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:22,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=68}] handler.RSProcedureHandler(53): pid=68 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:22,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=68 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=68 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:22,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/9a84a28639f14d8c9f09dcd386c3226c 2024-04-14T11:16:22,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/330568a701f14ce18dfacbf17088eaa2 is 50, key is test_row_0/C:col10/1713093381480/Put/seqid=0 2024-04-14T11:16:22,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/330568a701f14ce18dfacbf17088eaa2 2024-04-14T11:16:22,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/2b71dbcc28df4fd9a425065b9bdee307 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2b71dbcc28df4fd9a425065b9bdee307 2024-04-14T11:16:22,374 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:22,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=68 2024-04-14T11:16:22,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:22,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:22,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:22,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=68}] handler.RSProcedureHandler(53): pid=68 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:22,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=68 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=68 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:22,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2b71dbcc28df4fd9a425065b9bdee307, entries=150, sequenceid=52, filesize=11.7 K 2024-04-14T11:16:22,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/9a84a28639f14d8c9f09dcd386c3226c as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/9a84a28639f14d8c9f09dcd386c3226c 2024-04-14T11:16:22,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/9a84a28639f14d8c9f09dcd386c3226c, entries=150, sequenceid=52, filesize=11.7 K 2024-04-14T11:16:22,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/330568a701f14ce18dfacbf17088eaa2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/330568a701f14ce18dfacbf17088eaa2 2024-04-14T11:16:22,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/330568a701f14ce18dfacbf17088eaa2, entries=150, sequenceid=52, filesize=11.7 K 2024-04-14T11:16:22,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for c0901aa47578fba1a7f58734a4a0e3b2 in 922ms, sequenceid=52, compaction requested=true 2024-04-14T11:16:22,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:22,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:22,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:22,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:22,402 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:22,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:22,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:22,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:22,402 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:22,404 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:22,404 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/A is initiating minor compaction (all files) 2024-04-14T11:16:22,404 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:22,404 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/A in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:22,405 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/B is initiating minor compaction (all files) 2024-04-14T11:16:22,405 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/b5f0e269708f4170b0266c2ee661c20c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2dc3830742044f47b3f5ab267b3a1a41, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2b71dbcc28df4fd9a425065b9bdee307] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=35.1 K 2024-04-14T11:16:22,405 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/B in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:22,405 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/abd04d31efdb4bb98d5dda74c65288b0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/788dc4c85ec4450fabcb862c7411afc4, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/9a84a28639f14d8c9f09dcd386c3226c] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=35.1 K 2024-04-14T11:16:22,405 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting b5f0e269708f4170b0266c2ee661c20c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1713093380499 2024-04-14T11:16:22,406 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting abd04d31efdb4bb98d5dda74c65288b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1713093380499 2024-04-14T11:16:22,406 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 2dc3830742044f47b3f5ab267b3a1a41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1713093380535 2024-04-14T11:16:22,407 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 788dc4c85ec4450fabcb862c7411afc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1713093380535 2024-04-14T11:16:22,407 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 2b71dbcc28df4fd9a425065b9bdee307, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1713093380854 2024-04-14T11:16:22,408 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9a84a28639f14d8c9f09dcd386c3226c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1713093380854 2024-04-14T11:16:22,421 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#B#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:22,422 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/3cc1f600dbb042b0ac3468a0eeefb702 is 50, key is test_row_0/B:col10/1713093381480/Put/seqid=0 2024-04-14T11:16:22,424 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#A#compaction#224 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:22,425 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/815ab35dc91f4bf4abbe57600fe1c45d is 50, key is test_row_0/A:col10/1713093381480/Put/seqid=0 2024-04-14T11:16:22,528 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:22,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=68 2024-04-14T11:16:22,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:22,530 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-04-14T11:16:22,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/37b3eb2b2dc44c5fa5f7db11f06111dc is 50, key is test_row_0/A:col10/1713093381538/Put/seqid=0 2024-04-14T11:16:22,565 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/37b3eb2b2dc44c5fa5f7db11f06111dc 2024-04-14T11:16:22,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/04e29a382f9b4e57a1de33850a41cc1b is 50, key is test_row_0/B:col10/1713093381538/Put/seqid=0 2024-04-14T11:16:22,596 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/04e29a382f9b4e57a1de33850a41cc1b 2024-04-14T11:16:22,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/21cac03151784bb29892257f51888230 is 50, key is test_row_0/C:col10/1713093381538/Put/seqid=0 2024-04-14T11:16:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:22,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=67 2024-04-14T11:16:22,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:22,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093442729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:22,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093442729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093442729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:22,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093442731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:22,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093442731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:22,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093442839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:22,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:22,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093442839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093442839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:22,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093442839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:22,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093442839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:22,896 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/815ab35dc91f4bf4abbe57600fe1c45d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/815ab35dc91f4bf4abbe57600fe1c45d 2024-04-14T11:16:22,902 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/3cc1f600dbb042b0ac3468a0eeefb702 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/3cc1f600dbb042b0ac3468a0eeefb702 2024-04-14T11:16:22,912 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/A of c0901aa47578fba1a7f58734a4a0e3b2 into 815ab35dc91f4bf4abbe57600fe1c45d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:22,912 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:22,912 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/A, priority=13, startTime=1713093382402; duration=0sec 2024-04-14T11:16:22,912 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:22,912 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:A 2024-04-14T11:16:22,912 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:22,913 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/B of c0901aa47578fba1a7f58734a4a0e3b2 into 3cc1f600dbb042b0ac3468a0eeefb702(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:22,913 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:22,913 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/B, priority=13, startTime=1713093382402; duration=0sec 2024-04-14T11:16:22,913 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:22,913 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:B 2024-04-14T11:16:22,914 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:22,914 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/C is initiating minor compaction (all files) 2024-04-14T11:16:22,915 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/C in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:22,915 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/33f990f89a6044108dc8f3e9774fa980, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/23c9766ea79844c9a851602294bc32e6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/330568a701f14ce18dfacbf17088eaa2] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=35.1 K 2024-04-14T11:16:22,915 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 33f990f89a6044108dc8f3e9774fa980, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1713093380499 2024-04-14T11:16:22,916 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 23c9766ea79844c9a851602294bc32e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1713093380535 2024-04-14T11:16:22,916 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 330568a701f14ce18dfacbf17088eaa2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1713093380854 2024-04-14T11:16:22,940 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#C#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:22,940 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/823c64db2e1e4b1ea7e3d7587de75a78 is 50, key is test_row_0/C:col10/1713093381480/Put/seqid=0 2024-04-14T11:16:22,963 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/823c64db2e1e4b1ea7e3d7587de75a78 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/823c64db2e1e4b1ea7e3d7587de75a78 2024-04-14T11:16:22,973 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/C of c0901aa47578fba1a7f58734a4a0e3b2 into 823c64db2e1e4b1ea7e3d7587de75a78(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:22,973 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:22,973 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/C, priority=13, startTime=1713093382402; duration=0sec 2024-04-14T11:16:22,973 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:22,973 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:C 2024-04-14T11:16:23,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093443043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093443043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093443043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093443044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093443044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,050 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/21cac03151784bb29892257f51888230 2024-04-14T11:16:23,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/37b3eb2b2dc44c5fa5f7db11f06111dc as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/37b3eb2b2dc44c5fa5f7db11f06111dc 2024-04-14T11:16:23,071 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/37b3eb2b2dc44c5fa5f7db11f06111dc, entries=150, sequenceid=73, filesize=11.7 K 2024-04-14T11:16:23,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/04e29a382f9b4e57a1de33850a41cc1b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/04e29a382f9b4e57a1de33850a41cc1b 2024-04-14T11:16:23,084 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/04e29a382f9b4e57a1de33850a41cc1b, entries=150, sequenceid=73, filesize=11.7 K 2024-04-14T11:16:23,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/21cac03151784bb29892257f51888230 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/21cac03151784bb29892257f51888230 2024-04-14T11:16:23,094 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/21cac03151784bb29892257f51888230, entries=150, sequenceid=73, filesize=11.7 K 2024-04-14T11:16:23,095 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.HRegion(3035): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for c0901aa47578fba1a7f58734a4a0e3b2 in 565ms, sequenceid=73, compaction requested=false 2024-04-14T11:16:23,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:23,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:23,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=68}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=68 2024-04-14T11:16:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=68 2024-04-14T11:16:23,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=68, resume processing ppid=67 2024-04-14T11:16:23,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=68, ppid=67, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4860 sec 2024-04-14T11:16:23,103 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=67, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=67, table=TestAcidGuarantees in 1.4960 sec 2024-04-14T11:16:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:23,351 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:23,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-04-14T11:16:23,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/d25b9b0cb1114155af3bf581c0ccc75b is 50, key is test_row_0/A:col10/1713093383350/Put/seqid=0 2024-04-14T11:16:23,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093443386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093443388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093443388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093443390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093443389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093443494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093443495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093443496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093443496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093443501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,667 DEBUG [master/jenkins-hbase6:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 23980ec0107c2c55bc841b5f2502d0a1 changed from -1.0 to 0.0, refreshing cache 2024-04-14T11:16:23,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093443700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093443713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093443713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093443714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093443715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=67 2024-04-14T11:16:23,725 INFO [Thread-1198 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 67 completed 2024-04-14T11:16:23,728 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=69, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=69, table=TestAcidGuarantees 2024-04-14T11:16:23,733 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=69, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=69, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:23,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2024-04-14T11:16:23,734 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=69, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=69, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:23,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:23,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/d25b9b0cb1114155af3bf581c0ccc75b 2024-04-14T11:16:23,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/95ab2d9391b044b681fb0aa3d12ba808 is 50, key is test_row_0/B:col10/1713093383350/Put/seqid=0 2024-04-14T11:16:23,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/95ab2d9391b044b681fb0aa3d12ba808 2024-04-14T11:16:23,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/302c612a3a0345aca31d4f6540c791a1 is 50, key is test_row_0/C:col10/1713093383350/Put/seqid=0 2024-04-14T11:16:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2024-04-14T11:16:23,888 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:23,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=70 2024-04-14T11:16:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:23,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] handler.RSProcedureHandler(53): pid=70 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=70 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=70 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:24,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093444017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093444018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093444018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093444019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093444024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2024-04-14T11:16:24,042 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=70 2024-04-14T11:16:24,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:24,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:24,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:24,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=70}] handler.RSProcedureHandler(53): pid=70 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:24,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=70 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=70 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:24,196 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=70 2024-04-14T11:16:24,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:24,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:24,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:24,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=70}] handler.RSProcedureHandler(53): pid=70 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:24,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=70 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:24,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=70 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:24,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/302c612a3a0345aca31d4f6540c791a1 2024-04-14T11:16:24,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/d25b9b0cb1114155af3bf581c0ccc75b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/d25b9b0cb1114155af3bf581c0ccc75b 2024-04-14T11:16:24,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/d25b9b0cb1114155af3bf581c0ccc75b, entries=200, sequenceid=96, filesize=14.0 K 2024-04-14T11:16:24,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/95ab2d9391b044b681fb0aa3d12ba808 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/95ab2d9391b044b681fb0aa3d12ba808 2024-04-14T11:16:24,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/95ab2d9391b044b681fb0aa3d12ba808, entries=150, sequenceid=96, filesize=11.7 K 2024-04-14T11:16:24,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/302c612a3a0345aca31d4f6540c791a1 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/302c612a3a0345aca31d4f6540c791a1 2024-04-14T11:16:24,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/302c612a3a0345aca31d4f6540c791a1, entries=150, sequenceid=96, filesize=11.7 K 2024-04-14T11:16:24,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for c0901aa47578fba1a7f58734a4a0e3b2 in 930ms, sequenceid=96, compaction requested=true 2024-04-14T11:16:24,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:24,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:24,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:24,282 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:24,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:24,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:24,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:24,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-04-14T11:16:24,282 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:24,284 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38395 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:24,284 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/A is initiating minor compaction (all files) 2024-04-14T11:16:24,285 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/A in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:24,285 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/815ab35dc91f4bf4abbe57600fe1c45d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/37b3eb2b2dc44c5fa5f7db11f06111dc, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/d25b9b0cb1114155af3bf581c0ccc75b] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=37.5 K 2024-04-14T11:16:24,285 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36055 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:24,285 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/B is initiating minor compaction (all files) 2024-04-14T11:16:24,285 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/B in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:24,286 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 815ab35dc91f4bf4abbe57600fe1c45d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1713093380854 2024-04-14T11:16:24,286 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/3cc1f600dbb042b0ac3468a0eeefb702, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/04e29a382f9b4e57a1de33850a41cc1b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/95ab2d9391b044b681fb0aa3d12ba808] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=35.2 K 2024-04-14T11:16:24,286 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 37b3eb2b2dc44c5fa5f7db11f06111dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1713093381530 2024-04-14T11:16:24,287 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 3cc1f600dbb042b0ac3468a0eeefb702, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1713093380854 2024-04-14T11:16:24,288 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting d25b9b0cb1114155af3bf581c0ccc75b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1713093382728 2024-04-14T11:16:24,288 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 04e29a382f9b4e57a1de33850a41cc1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1713093381530 2024-04-14T11:16:24,289 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 95ab2d9391b044b681fb0aa3d12ba808, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1713093383348 2024-04-14T11:16:24,302 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#A#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:24,303 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/5f8f85adda584a2e8e4e7d90ae2effe4 is 50, key is test_row_0/A:col10/1713093383350/Put/seqid=0 2024-04-14T11:16:24,309 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#B#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:24,310 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/d1e35a799d6b4cc486d97ca00857dba6 is 50, key is test_row_0/B:col10/1713093383350/Put/seqid=0 2024-04-14T11:16:24,325 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/5f8f85adda584a2e8e4e7d90ae2effe4 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5f8f85adda584a2e8e4e7d90ae2effe4 2024-04-14T11:16:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2024-04-14T11:16:24,340 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/A of c0901aa47578fba1a7f58734a4a0e3b2 into 5f8f85adda584a2e8e4e7d90ae2effe4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:24,340 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:24,340 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/A, priority=13, startTime=1713093384282; duration=0sec 2024-04-14T11:16:24,340 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:24,340 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:A 2024-04-14T11:16:24,340 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:24,342 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36055 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:24,342 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/C is initiating minor compaction (all files) 2024-04-14T11:16:24,342 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/C in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:24,342 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/823c64db2e1e4b1ea7e3d7587de75a78, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/21cac03151784bb29892257f51888230, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/302c612a3a0345aca31d4f6540c791a1] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=35.2 K 2024-04-14T11:16:24,343 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 823c64db2e1e4b1ea7e3d7587de75a78, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1713093380854 2024-04-14T11:16:24,344 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 21cac03151784bb29892257f51888230, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1713093381530 2024-04-14T11:16:24,344 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 302c612a3a0345aca31d4f6540c791a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1713093383348 2024-04-14T11:16:24,349 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=70 2024-04-14T11:16:24,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:24,350 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-04-14T11:16:24,378 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#C#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:24,379 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/c1f412b242fc48698bd32eaffc9b5300 is 50, key is test_row_0/C:col10/1713093383350/Put/seqid=0 2024-04-14T11:16:24,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/cfc84c13ae7a46e6b8a4f2d9c8eb1630 is 50, key is test_row_0/A:col10/1713093383379/Put/seqid=0 2024-04-14T11:16:24,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:24,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:24,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093444542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093444542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093444543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093444544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093444546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,553 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-04-14T11:16:24,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093444647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093444647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093444649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093444649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093444652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,742 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/d1e35a799d6b4cc486d97ca00857dba6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/d1e35a799d6b4cc486d97ca00857dba6 2024-04-14T11:16:24,751 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/B of c0901aa47578fba1a7f58734a4a0e3b2 into d1e35a799d6b4cc486d97ca00857dba6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:24,751 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:24,751 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/B, priority=13, startTime=1713093384282; duration=0sec 2024-04-14T11:16:24,751 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:24,751 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:B 2024-04-14T11:16:24,817 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/cfc84c13ae7a46e6b8a4f2d9c8eb1630 2024-04-14T11:16:24,819 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/c1f412b242fc48698bd32eaffc9b5300 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c1f412b242fc48698bd32eaffc9b5300 2024-04-14T11:16:24,828 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/C of c0901aa47578fba1a7f58734a4a0e3b2 into c1f412b242fc48698bd32eaffc9b5300(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:24,828 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:24,828 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/C, priority=13, startTime=1713093384282; duration=0sec 2024-04-14T11:16:24,828 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:24,828 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:C 2024-04-14T11:16:24,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/a3e8835ba40c4bb394e720fdd0df479c is 50, key is test_row_0/B:col10/1713093383379/Put/seqid=0 2024-04-14T11:16:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2024-04-14T11:16:24,842 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/a3e8835ba40c4bb394e720fdd0df479c 2024-04-14T11:16:24,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093444851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093444852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093444852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093444853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:24,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/2baa9cc6ae7a42a892b42bcdad7da609 is 50, key is test_row_0/C:col10/1713093383379/Put/seqid=0 2024-04-14T11:16:24,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093444856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093445153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093445156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093445157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093445158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093445158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,264 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/2baa9cc6ae7a42a892b42bcdad7da609 2024-04-14T11:16:25,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/cfc84c13ae7a46e6b8a4f2d9c8eb1630 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/cfc84c13ae7a46e6b8a4f2d9c8eb1630 2024-04-14T11:16:25,288 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/cfc84c13ae7a46e6b8a4f2d9c8eb1630, entries=150, sequenceid=114, filesize=11.7 K 2024-04-14T11:16:25,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/a3e8835ba40c4bb394e720fdd0df479c as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/a3e8835ba40c4bb394e720fdd0df479c 2024-04-14T11:16:25,297 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/a3e8835ba40c4bb394e720fdd0df479c, entries=150, sequenceid=114, filesize=11.7 K 2024-04-14T11:16:25,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/2baa9cc6ae7a42a892b42bcdad7da609 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2baa9cc6ae7a42a892b42bcdad7da609 2024-04-14T11:16:25,312 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2baa9cc6ae7a42a892b42bcdad7da609, entries=150, sequenceid=114, filesize=11.7 K 2024-04-14T11:16:25,314 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.HRegion(3035): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for c0901aa47578fba1a7f58734a4a0e3b2 in 964ms, sequenceid=114, compaction requested=false 2024-04-14T11:16:25,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:25,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:25,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=70}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=70 2024-04-14T11:16:25,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=70 2024-04-14T11:16:25,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=70, resume processing ppid=69 2024-04-14T11:16:25,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=70, ppid=69, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5810 sec 2024-04-14T11:16:25,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=69, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=69, table=TestAcidGuarantees in 1.6050 sec 2024-04-14T11:16:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:25,660 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:25,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-04-14T11:16:25,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/5ad935177c7f4904b4d37f272e939792 is 50, key is test_row_0/A:col10/1713093384537/Put/seqid=0 2024-04-14T11:16:25,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093445673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093445674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093445676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093445676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093445677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/5ad935177c7f4904b4d37f272e939792 2024-04-14T11:16:25,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/f6daee50e0eb453b9e280eb3416e25ed is 50, key is test_row_0/B:col10/1713093384537/Put/seqid=0 2024-04-14T11:16:25,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093445777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093445778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093445781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093445781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093445781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=69 2024-04-14T11:16:25,842 INFO [Thread-1198 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 69 completed 2024-04-14T11:16:25,847 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:25,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=71, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=71, table=TestAcidGuarantees 2024-04-14T11:16:25,851 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=71, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=71, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2024-04-14T11:16:25,852 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=71, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=71, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:25,852 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:25,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2024-04-14T11:16:25,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093445981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093445982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093445984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093445985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:25,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093445985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,004 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=72 2024-04-14T11:16:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:26,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] handler.RSProcedureHandler(53): pid=72 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=72 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=72 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:26,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/f6daee50e0eb453b9e280eb3416e25ed 2024-04-14T11:16:26,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/d0d90505773c48afaa225faf8f0bbacc is 50, key is test_row_0/C:col10/1713093384537/Put/seqid=0 2024-04-14T11:16:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2024-04-14T11:16:26,157 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=72 2024-04-14T11:16:26,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:26,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:26,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:26,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] handler.RSProcedureHandler(53): pid=72 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:26,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=72 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=72 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:26,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093446285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093446286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093446287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093446288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093446288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,310 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=72 2024-04-14T11:16:26,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:26,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:26,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:26,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] handler.RSProcedureHandler(53): pid=72 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:26,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=72 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=72 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2024-04-14T11:16:26,464 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=72 2024-04-14T11:16:26,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:26,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:26,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:26,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] handler.RSProcedureHandler(53): pid=72 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:26,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=72 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=72 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:26,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/d0d90505773c48afaa225faf8f0bbacc 2024-04-14T11:16:26,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/5ad935177c7f4904b4d37f272e939792 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5ad935177c7f4904b4d37f272e939792 2024-04-14T11:16:26,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5ad935177c7f4904b4d37f272e939792, entries=200, sequenceid=136, filesize=14.1 K 2024-04-14T11:16:26,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/f6daee50e0eb453b9e280eb3416e25ed as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f6daee50e0eb453b9e280eb3416e25ed 2024-04-14T11:16:26,617 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=72 2024-04-14T11:16:26,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:26,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:26,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:26,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] handler.RSProcedureHandler(53): pid=72 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:26,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=72 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:26,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f6daee50e0eb453b9e280eb3416e25ed, entries=150, sequenceid=136, filesize=11.8 K 2024-04-14T11:16:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=72 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:26,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/d0d90505773c48afaa225faf8f0bbacc as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/d0d90505773c48afaa225faf8f0bbacc 2024-04-14T11:16:26,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/d0d90505773c48afaa225faf8f0bbacc, entries=150, sequenceid=136, filesize=11.8 K 2024-04-14T11:16:26,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for c0901aa47578fba1a7f58734a4a0e3b2 in 968ms, sequenceid=136, compaction requested=true 2024-04-14T11:16:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:26,628 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:26,628 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:26,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:26,630 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38648 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:26,630 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/A is initiating minor compaction (all files) 2024-04-14T11:16:26,630 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/A in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:26,630 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5f8f85adda584a2e8e4e7d90ae2effe4, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/cfc84c13ae7a46e6b8a4f2d9c8eb1630, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5ad935177c7f4904b4d37f272e939792] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=37.7 K 2024-04-14T11:16:26,631 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36258 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:26,631 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/B is initiating minor compaction (all files) 2024-04-14T11:16:26,631 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/B in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:26,631 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/d1e35a799d6b4cc486d97ca00857dba6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/a3e8835ba40c4bb394e720fdd0df479c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f6daee50e0eb453b9e280eb3416e25ed] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=35.4 K 2024-04-14T11:16:26,631 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5f8f85adda584a2e8e4e7d90ae2effe4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1713093383348 2024-04-14T11:16:26,632 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting d1e35a799d6b4cc486d97ca00857dba6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1713093383348 2024-04-14T11:16:26,632 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting cfc84c13ae7a46e6b8a4f2d9c8eb1630, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1713093383379 2024-04-14T11:16:26,632 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting a3e8835ba40c4bb394e720fdd0df479c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1713093383379 2024-04-14T11:16:26,632 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5ad935177c7f4904b4d37f272e939792, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1713093384537 2024-04-14T11:16:26,633 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting f6daee50e0eb453b9e280eb3416e25ed, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1713093384537 2024-04-14T11:16:26,643 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#B#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:26,644 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/beb94d3fec3c44edb2e7fd854eeede31 is 50, key is test_row_0/B:col10/1713093384537/Put/seqid=0 2024-04-14T11:16:26,663 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#A#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:26,663 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/c54ef7815aa942318e05332e54204a8d is 50, key is test_row_0/A:col10/1713093384537/Put/seqid=0 2024-04-14T11:16:26,679 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/beb94d3fec3c44edb2e7fd854eeede31 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/beb94d3fec3c44edb2e7fd854eeede31 2024-04-14T11:16:26,687 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/c54ef7815aa942318e05332e54204a8d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/c54ef7815aa942318e05332e54204a8d 2024-04-14T11:16:26,694 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/B of c0901aa47578fba1a7f58734a4a0e3b2 into beb94d3fec3c44edb2e7fd854eeede31(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:26,694 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:26,694 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/B, priority=13, startTime=1713093386628; duration=0sec 2024-04-14T11:16:26,694 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:26,694 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:B 2024-04-14T11:16:26,694 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:26,697 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36258 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:26,697 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/C is initiating minor compaction (all files) 2024-04-14T11:16:26,697 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/C in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:26,697 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c1f412b242fc48698bd32eaffc9b5300, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2baa9cc6ae7a42a892b42bcdad7da609, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/d0d90505773c48afaa225faf8f0bbacc] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=35.4 K 2024-04-14T11:16:26,699 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting c1f412b242fc48698bd32eaffc9b5300, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1713093383348 2024-04-14T11:16:26,699 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/A of c0901aa47578fba1a7f58734a4a0e3b2 into c54ef7815aa942318e05332e54204a8d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:26,699 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:26,699 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/A, priority=13, startTime=1713093386628; duration=0sec 2024-04-14T11:16:26,699 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:26,699 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:A 2024-04-14T11:16:26,699 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 2baa9cc6ae7a42a892b42bcdad7da609, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1713093383379 2024-04-14T11:16:26,700 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting d0d90505773c48afaa225faf8f0bbacc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1713093384537 2024-04-14T11:16:26,712 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#C#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:26,713 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/a027c7702b604ab29b1f8196a7bfa92b is 50, key is test_row_0/C:col10/1713093384537/Put/seqid=0 2024-04-14T11:16:26,738 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/a027c7702b604ab29b1f8196a7bfa92b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a027c7702b604ab29b1f8196a7bfa92b 2024-04-14T11:16:26,751 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/C of c0901aa47578fba1a7f58734a4a0e3b2 into a027c7702b604ab29b1f8196a7bfa92b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:26,751 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:26,751 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/C, priority=13, startTime=1713093386628; duration=0sec 2024-04-14T11:16:26,751 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:26,751 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:C 2024-04-14T11:16:26,773 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=72 2024-04-14T11:16:26,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:26,774 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-04-14T11:16:26,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/709804e770ec450c8b996438f12e07d3 is 50, key is test_row_0/A:col10/1713093385674/Put/seqid=0 2024-04-14T11:16:26,787 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/709804e770ec450c8b996438f12e07d3 2024-04-14T11:16:26,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:26,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/8f8779931cdd404b8c88a8f1cb91eb50 is 50, key is test_row_0/B:col10/1713093385674/Put/seqid=0 2024-04-14T11:16:26,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:26,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093446811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093446812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093446814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093446815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093446817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093446918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:26,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093446919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093446919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093446920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093446921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2024-04-14T11:16:27,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093447123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093447125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093447125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093447125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093447128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,218 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/8f8779931cdd404b8c88a8f1cb91eb50 2024-04-14T11:16:27,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/3ce8feaa080741d6835a0bf425f4c6c6 is 50, key is test_row_0/C:col10/1713093385674/Put/seqid=0 2024-04-14T11:16:27,257 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/3ce8feaa080741d6835a0bf425f4c6c6 2024-04-14T11:16:27,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/709804e770ec450c8b996438f12e07d3 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/709804e770ec450c8b996438f12e07d3 2024-04-14T11:16:27,275 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/709804e770ec450c8b996438f12e07d3, entries=150, sequenceid=155, filesize=11.8 K 2024-04-14T11:16:27,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/8f8779931cdd404b8c88a8f1cb91eb50 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/8f8779931cdd404b8c88a8f1cb91eb50 2024-04-14T11:16:27,284 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/8f8779931cdd404b8c88a8f1cb91eb50, entries=150, sequenceid=155, filesize=11.8 K 2024-04-14T11:16:27,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/3ce8feaa080741d6835a0bf425f4c6c6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/3ce8feaa080741d6835a0bf425f4c6c6 2024-04-14T11:16:27,299 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/3ce8feaa080741d6835a0bf425f4c6c6, entries=150, sequenceid=155, filesize=11.8 K 2024-04-14T11:16:27,301 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(3035): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for c0901aa47578fba1a7f58734a4a0e3b2 in 527ms, sequenceid=155, compaction requested=false 2024-04-14T11:16:27,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:27,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:27,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=72 2024-04-14T11:16:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=72 2024-04-14T11:16:27,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=72, resume processing ppid=71 2024-04-14T11:16:27,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=72, ppid=71, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4500 sec 2024-04-14T11:16:27,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=71, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=71, table=TestAcidGuarantees in 1.4750 sec 2024-04-14T11:16:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:27,427 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:27,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-04-14T11:16:27,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/5bc39d8ddc484a6cb9973b21cd311467 is 50, key is test_row_0/A:col10/1713093387425/Put/seqid=0 2024-04-14T11:16:27,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/5bc39d8ddc484a6cb9973b21cd311467 2024-04-14T11:16:27,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/f855465c00404beeaae3d59dc8253de3 is 50, key is test_row_0/B:col10/1713093387425/Put/seqid=0 2024-04-14T11:16:27,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093447441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093447443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/f855465c00404beeaae3d59dc8253de3 2024-04-14T11:16:27,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093447456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093447467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093447467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/ae5cd44b9fa74344b7ad0cd23e7c48f9 is 50, key is test_row_0/C:col10/1713093387425/Put/seqid=0 2024-04-14T11:16:27,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/ae5cd44b9fa74344b7ad0cd23e7c48f9 2024-04-14T11:16:27,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/5bc39d8ddc484a6cb9973b21cd311467 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5bc39d8ddc484a6cb9973b21cd311467 2024-04-14T11:16:27,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5bc39d8ddc484a6cb9973b21cd311467, entries=150, sequenceid=176, filesize=11.8 K 2024-04-14T11:16:27,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/f855465c00404beeaae3d59dc8253de3 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f855465c00404beeaae3d59dc8253de3 2024-04-14T11:16:27,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f855465c00404beeaae3d59dc8253de3, entries=150, sequenceid=176, filesize=11.8 K 2024-04-14T11:16:27,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/ae5cd44b9fa74344b7ad0cd23e7c48f9 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/ae5cd44b9fa74344b7ad0cd23e7c48f9 2024-04-14T11:16:27,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/ae5cd44b9fa74344b7ad0cd23e7c48f9, entries=150, sequenceid=176, filesize=11.8 K 2024-04-14T11:16:27,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=93.93 KB/96180 for c0901aa47578fba1a7f58734a4a0e3b2 in 125ms, sequenceid=176, compaction requested=true 2024-04-14T11:16:27,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:27,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:27,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:27,552 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:27,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:27,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:27,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:27,552 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:27,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:27,554 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36660 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:27,555 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/A is initiating minor compaction (all files) 2024-04-14T11:16:27,555 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/A in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:27,555 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36660 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:27,555 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/c54ef7815aa942318e05332e54204a8d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/709804e770ec450c8b996438f12e07d3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5bc39d8ddc484a6cb9973b21cd311467] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=35.8 K 2024-04-14T11:16:27,555 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/B is initiating minor compaction (all files) 2024-04-14T11:16:27,555 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/B in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:27,555 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/beb94d3fec3c44edb2e7fd854eeede31, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/8f8779931cdd404b8c88a8f1cb91eb50, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f855465c00404beeaae3d59dc8253de3] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=35.8 K 2024-04-14T11:16:27,555 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c54ef7815aa942318e05332e54204a8d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1713093384537 2024-04-14T11:16:27,556 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting beb94d3fec3c44edb2e7fd854eeede31, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1713093384537 2024-04-14T11:16:27,557 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 709804e770ec450c8b996438f12e07d3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1713093385671 2024-04-14T11:16:27,557 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 8f8779931cdd404b8c88a8f1cb91eb50, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1713093385671 2024-04-14T11:16:27,557 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5bc39d8ddc484a6cb9973b21cd311467, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1713093386790 2024-04-14T11:16:27,558 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting f855465c00404beeaae3d59dc8253de3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1713093386790 2024-04-14T11:16:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:27,560 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:27,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-04-14T11:16:27,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/5527ee4b31e143438793ef0c9931e0bb is 50, key is test_row_0/A:col10/1713093387558/Put/seqid=0 2024-04-14T11:16:27,584 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#A#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:27,584 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/332f364af45c42c2a1f20db1f7d9405b is 50, key is test_row_0/A:col10/1713093387425/Put/seqid=0 2024-04-14T11:16:27,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093447580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093447581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093447583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093447586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093447586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,595 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:27,595 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/5d5bd0f8a39b476d970be6b4dd0d5baa is 50, key is test_row_0/B:col10/1713093387425/Put/seqid=0 2024-04-14T11:16:27,666 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/332f364af45c42c2a1f20db1f7d9405b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/332f364af45c42c2a1f20db1f7d9405b 2024-04-14T11:16:27,673 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/5d5bd0f8a39b476d970be6b4dd0d5baa as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/5d5bd0f8a39b476d970be6b4dd0d5baa 2024-04-14T11:16:27,675 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/A of c0901aa47578fba1a7f58734a4a0e3b2 into 332f364af45c42c2a1f20db1f7d9405b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:27,675 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:27,675 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/A, priority=13, startTime=1713093387552; duration=0sec 2024-04-14T11:16:27,675 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:27,675 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:A 2024-04-14T11:16:27,675 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:27,678 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36660 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:27,678 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/C is initiating minor compaction (all files) 2024-04-14T11:16:27,678 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/C in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:27,679 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a027c7702b604ab29b1f8196a7bfa92b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/3ce8feaa080741d6835a0bf425f4c6c6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/ae5cd44b9fa74344b7ad0cd23e7c48f9] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=35.8 K 2024-04-14T11:16:27,680 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting a027c7702b604ab29b1f8196a7bfa92b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1713093384537 2024-04-14T11:16:27,680 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3ce8feaa080741d6835a0bf425f4c6c6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1713093385671 2024-04-14T11:16:27,681 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting ae5cd44b9fa74344b7ad0cd23e7c48f9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1713093386790 2024-04-14T11:16:27,681 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/B of c0901aa47578fba1a7f58734a4a0e3b2 into 5d5bd0f8a39b476d970be6b4dd0d5baa(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:27,681 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:27,681 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/B, priority=13, startTime=1713093387552; duration=0sec 2024-04-14T11:16:27,681 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:27,681 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:B 2024-04-14T11:16:27,692 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#C#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:27,692 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/b1978d58ff52484fa3cd4a2ca26d7aef is 50, key is test_row_0/C:col10/1713093387425/Put/seqid=0 2024-04-14T11:16:27,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093447687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093447688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093447688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093447691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093447691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093447899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093447899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093447899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093447900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093447909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=71 2024-04-14T11:16:27,967 INFO [Thread-1198 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 71 completed 2024-04-14T11:16:27,979 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=73, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=73, table=TestAcidGuarantees 2024-04-14T11:16:27,996 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=73, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=73, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=73 2024-04-14T11:16:27,999 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=73, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=73, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:27,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:28,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/5527ee4b31e143438793ef0c9931e0bb 2024-04-14T11:16:28,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/3d7401081d6c452b9ba6c5ab204373f3 is 50, key is test_row_0/B:col10/1713093387558/Put/seqid=0 2024-04-14T11:16:28,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/3d7401081d6c452b9ba6c5ab204373f3 2024-04-14T11:16:28,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/a364dc55e8aa43b68b202b508544d40d is 50, key is test_row_0/C:col10/1713093387558/Put/seqid=0 2024-04-14T11:16:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=73 2024-04-14T11:16:28,147 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/b1978d58ff52484fa3cd4a2ca26d7aef as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/b1978d58ff52484fa3cd4a2ca26d7aef 2024-04-14T11:16:28,151 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:28,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=74 2024-04-14T11:16:28,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:28,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:28,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:28,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] handler.RSProcedureHandler(53): pid=74 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:28,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=74 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=74 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:28,157 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/C of c0901aa47578fba1a7f58734a4a0e3b2 into b1978d58ff52484fa3cd4a2ca26d7aef(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:28,157 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:28,157 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/C, priority=13, startTime=1713093387552; duration=0sec 2024-04-14T11:16:28,157 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:28,157 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:C 2024-04-14T11:16:28,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093448203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:28,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:28,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093448203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:28,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093448205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:28,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093448205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:28,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093448213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=73 2024-04-14T11:16:28,306 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:28,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=74 2024-04-14T11:16:28,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:28,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:28,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:28,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] handler.RSProcedureHandler(53): pid=74 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:28,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=74 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=74 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:28,460 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:28,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=74 2024-04-14T11:16:28,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:28,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:28,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:28,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] handler.RSProcedureHandler(53): pid=74 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:28,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=74 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=74 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:28,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/a364dc55e8aa43b68b202b508544d40d 2024-04-14T11:16:28,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/5527ee4b31e143438793ef0c9931e0bb as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5527ee4b31e143438793ef0c9931e0bb 2024-04-14T11:16:28,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5527ee4b31e143438793ef0c9931e0bb, entries=200, sequenceid=194, filesize=14.2 K 2024-04-14T11:16:28,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/3d7401081d6c452b9ba6c5ab204373f3 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/3d7401081d6c452b9ba6c5ab204373f3 2024-04-14T11:16:28,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/3d7401081d6c452b9ba6c5ab204373f3, entries=150, sequenceid=194, filesize=11.8 K 2024-04-14T11:16:28,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/a364dc55e8aa43b68b202b508544d40d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a364dc55e8aa43b68b202b508544d40d 2024-04-14T11:16:28,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a364dc55e8aa43b68b202b508544d40d, entries=150, sequenceid=194, filesize=11.8 K 2024-04-14T11:16:28,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for c0901aa47578fba1a7f58734a4a0e3b2 in 977ms, sequenceid=194, compaction requested=false 2024-04-14T11:16:28,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=73 2024-04-14T11:16:28,616 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:28,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=74 2024-04-14T11:16:28,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:28,617 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-04-14T11:16:28,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/52c5cfd07d134cf8bce3913aa1fc75db is 50, key is test_row_0/A:col10/1713093387579/Put/seqid=0 2024-04-14T11:16:28,649 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/52c5cfd07d134cf8bce3913aa1fc75db 2024-04-14T11:16:28,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/a5e26fa4198141a9b3b0d132e45dbd42 is 50, key is test_row_0/B:col10/1713093387579/Put/seqid=0 2024-04-14T11:16:28,708 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/a5e26fa4198141a9b3b0d132e45dbd42 2024-04-14T11:16:28,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:28,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/a55c6a042585461e8f2bfb7209e727fc is 50, key is test_row_0/C:col10/1713093387579/Put/seqid=0 2024-04-14T11:16:28,745 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/a55c6a042585461e8f2bfb7209e727fc 2024-04-14T11:16:28,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093448738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:28,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093448740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:28,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093448745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:28,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093448749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:28,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093448750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:28,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/52c5cfd07d134cf8bce3913aa1fc75db as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/52c5cfd07d134cf8bce3913aa1fc75db 2024-04-14T11:16:28,775 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/52c5cfd07d134cf8bce3913aa1fc75db, entries=150, sequenceid=215, filesize=11.8 K 2024-04-14T11:16:28,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/a5e26fa4198141a9b3b0d132e45dbd42 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/a5e26fa4198141a9b3b0d132e45dbd42 2024-04-14T11:16:28,798 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/a5e26fa4198141a9b3b0d132e45dbd42, entries=150, sequenceid=215, filesize=11.8 K 2024-04-14T11:16:28,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/a55c6a042585461e8f2bfb7209e727fc as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a55c6a042585461e8f2bfb7209e727fc 2024-04-14T11:16:28,809 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a55c6a042585461e8f2bfb7209e727fc, entries=150, sequenceid=215, filesize=11.8 K 2024-04-14T11:16:28,810 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(3035): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for c0901aa47578fba1a7f58734a4a0e3b2 in 193ms, sequenceid=215, compaction requested=true 2024-04-14T11:16:28,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:28,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:28,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=74 2024-04-14T11:16:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=74 2024-04-14T11:16:28,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=74, resume processing ppid=73 2024-04-14T11:16:28,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=74, ppid=73, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 812 msec 2024-04-14T11:16:28,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=73, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=73, table=TestAcidGuarantees in 836 msec 2024-04-14T11:16:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:28,855 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:28,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-04-14T11:16:28,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/224e720eaea245e4ba45547f7d495cda is 50, key is test_row_0/A:col10/1713093388854/Put/seqid=0 2024-04-14T11:16:28,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/224e720eaea245e4ba45547f7d495cda 2024-04-14T11:16:28,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:28,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093448920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:28,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093448921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:28,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:28,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093448923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093448921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:28,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:28,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093448928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:28,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/be340246078e4e65a961087e1428fa5b is 50, key is test_row_0/B:col10/1713093388854/Put/seqid=0 2024-04-14T11:16:29,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093449029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093449029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093449030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093449030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093449034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=73 2024-04-14T11:16:29,104 INFO [Thread-1198 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 73 completed 2024-04-14T11:16:29,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=75, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees 2024-04-14T11:16:29,112 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=75, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:29,113 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=75, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:29,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=75 2024-04-14T11:16:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=75 2024-04-14T11:16:29,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093449236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093449237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093449238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093449238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093449239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,265 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-04-14T11:16:29,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:29,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:29,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:29,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(53): pid=76 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:29,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:29,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/be340246078e4e65a961087e1428fa5b 2024-04-14T11:16:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=75 2024-04-14T11:16:29,420 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-04-14T11:16:29,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:29,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:29,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:29,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(53): pid=76 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:29,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:29,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/c4a6cd29faac4cd9b80d1a0290509e55 is 50, key is test_row_0/C:col10/1713093388854/Put/seqid=0 2024-04-14T11:16:29,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/c4a6cd29faac4cd9b80d1a0290509e55 2024-04-14T11:16:29,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/224e720eaea245e4ba45547f7d495cda as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/224e720eaea245e4ba45547f7d495cda 2024-04-14T11:16:29,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/224e720eaea245e4ba45547f7d495cda, entries=200, sequenceid=235, filesize=14.2 K 2024-04-14T11:16:29,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/be340246078e4e65a961087e1428fa5b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/be340246078e4e65a961087e1428fa5b 2024-04-14T11:16:29,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/be340246078e4e65a961087e1428fa5b, entries=150, sequenceid=235, filesize=11.8 K 2024-04-14T11:16:29,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/c4a6cd29faac4cd9b80d1a0290509e55 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c4a6cd29faac4cd9b80d1a0290509e55 2024-04-14T11:16:29,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c4a6cd29faac4cd9b80d1a0290509e55, entries=150, sequenceid=235, filesize=11.8 K 2024-04-14T11:16:29,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for c0901aa47578fba1a7f58734a4a0e3b2 in 647ms, sequenceid=235, compaction requested=true 2024-04-14T11:16:29,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:29,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:29,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:29,504 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:29,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:29,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:29,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:29,504 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:29,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:29,507 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:29,507 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/A is initiating minor compaction (all files) 2024-04-14T11:16:29,507 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/A in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:29,507 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/332f364af45c42c2a1f20db1f7d9405b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5527ee4b31e143438793ef0c9931e0bb, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/52c5cfd07d134cf8bce3913aa1fc75db, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/224e720eaea245e4ba45547f7d495cda] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=52.5 K 2024-04-14T11:16:29,508 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:29,508 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/B is initiating minor compaction (all files) 2024-04-14T11:16:29,508 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/B in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:29,508 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/5d5bd0f8a39b476d970be6b4dd0d5baa, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/3d7401081d6c452b9ba6c5ab204373f3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/a5e26fa4198141a9b3b0d132e45dbd42, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/be340246078e4e65a961087e1428fa5b] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=47.8 K 2024-04-14T11:16:29,508 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 332f364af45c42c2a1f20db1f7d9405b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1713093386790 2024-04-14T11:16:29,509 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5d5bd0f8a39b476d970be6b4dd0d5baa, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1713093386790 2024-04-14T11:16:29,510 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 5527ee4b31e143438793ef0c9931e0bb, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1713093387442 2024-04-14T11:16:29,510 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3d7401081d6c452b9ba6c5ab204373f3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1713093387442 2024-04-14T11:16:29,510 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 52c5cfd07d134cf8bce3913aa1fc75db, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1713093387579 2024-04-14T11:16:29,511 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting a5e26fa4198141a9b3b0d132e45dbd42, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1713093387579 2024-04-14T11:16:29,511 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 224e720eaea245e4ba45547f7d495cda, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1713093388735 2024-04-14T11:16:29,511 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting be340246078e4e65a961087e1428fa5b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1713093388735 2024-04-14T11:16:29,538 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#A#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:29,539 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/c9264ba28c884b689b4f63fe2935dd1b is 50, key is test_row_0/A:col10/1713093388854/Put/seqid=0 2024-04-14T11:16:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:29,551 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:29,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-04-14T11:16:29,551 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#B#compaction#263 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:29,551 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/f87225c8929942abb75fd4010c2b2ad7 is 50, key is test_row_0/B:col10/1713093388854/Put/seqid=0 2024-04-14T11:16:29,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/df1526798ebd457b9b43f3a36f833639 is 50, key is test_row_0/A:col10/1713093388879/Put/seqid=0 2024-04-14T11:16:29,574 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-04-14T11:16:29,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:29,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:29,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:29,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(53): pid=76 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:29,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:29,601 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/c9264ba28c884b689b4f63fe2935dd1b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/c9264ba28c884b689b4f63fe2935dd1b 2024-04-14T11:16:29,615 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/f87225c8929942abb75fd4010c2b2ad7 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f87225c8929942abb75fd4010c2b2ad7 2024-04-14T11:16:29,636 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/B of c0901aa47578fba1a7f58734a4a0e3b2 into f87225c8929942abb75fd4010c2b2ad7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:29,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/df1526798ebd457b9b43f3a36f833639 2024-04-14T11:16:29,636 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:29,636 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/B, priority=12, startTime=1713093389504; duration=0sec 2024-04-14T11:16:29,637 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/A of c0901aa47578fba1a7f58734a4a0e3b2 into c9264ba28c884b689b4f63fe2935dd1b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:29,637 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:29,637 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/A, priority=12, startTime=1713093389504; duration=0sec 2024-04-14T11:16:29,637 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:29,637 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:B 2024-04-14T11:16:29,637 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:29,637 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:29,637 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:A 2024-04-14T11:16:29,641 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:29,641 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/C is initiating minor compaction (all files) 2024-04-14T11:16:29,641 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/C in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:29,641 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/b1978d58ff52484fa3cd4a2ca26d7aef, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a364dc55e8aa43b68b202b508544d40d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a55c6a042585461e8f2bfb7209e727fc, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c4a6cd29faac4cd9b80d1a0290509e55] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=47.8 K 2024-04-14T11:16:29,642 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting b1978d58ff52484fa3cd4a2ca26d7aef, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1713093386790 2024-04-14T11:16:29,643 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting a364dc55e8aa43b68b202b508544d40d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1713093387442 2024-04-14T11:16:29,643 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting a55c6a042585461e8f2bfb7209e727fc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1713093387579 2024-04-14T11:16:29,644 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c4a6cd29faac4cd9b80d1a0290509e55, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1713093388735 2024-04-14T11:16:29,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093449641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093449640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/840b2decca8946b1a065decae005847e is 50, key is test_row_0/B:col10/1713093388879/Put/seqid=0 2024-04-14T11:16:29,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093449655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093449655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093449656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,672 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#C#compaction#266 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:29,673 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/a0c6e3ef70b54436b24f95419251f811 is 50, key is test_row_0/C:col10/1713093388854/Put/seqid=0 2024-04-14T11:16:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=75 2024-04-14T11:16:29,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/840b2decca8946b1a065decae005847e 2024-04-14T11:16:29,730 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-04-14T11:16:29,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:29,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:29,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:29,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(53): pid=76 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:29,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:29,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/ccacfd255932421bbec7550f0a788503 is 50, key is test_row_0/C:col10/1713093388879/Put/seqid=0 2024-04-14T11:16:29,750 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/a0c6e3ef70b54436b24f95419251f811 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a0c6e3ef70b54436b24f95419251f811 2024-04-14T11:16:29,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093449757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093449757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,764 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/C of c0901aa47578fba1a7f58734a4a0e3b2 into a0c6e3ef70b54436b24f95419251f811(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:29,764 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:29,764 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/C, priority=12, startTime=1713093389504; duration=0sec 2024-04-14T11:16:29,764 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:29,764 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:C 2024-04-14T11:16:29,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093449762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093449762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093449763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/ccacfd255932421bbec7550f0a788503 2024-04-14T11:16:29,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/df1526798ebd457b9b43f3a36f833639 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/df1526798ebd457b9b43f3a36f833639 2024-04-14T11:16:29,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/df1526798ebd457b9b43f3a36f833639, entries=300, sequenceid=254, filesize=18.9 K 2024-04-14T11:16:29,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/840b2decca8946b1a065decae005847e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/840b2decca8946b1a065decae005847e 2024-04-14T11:16:29,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/840b2decca8946b1a065decae005847e, entries=150, sequenceid=254, filesize=11.8 K 2024-04-14T11:16:29,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/ccacfd255932421bbec7550f0a788503 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/ccacfd255932421bbec7550f0a788503 2024-04-14T11:16:29,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/ccacfd255932421bbec7550f0a788503, entries=150, sequenceid=254, filesize=11.8 K 2024-04-14T11:16:29,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=107.34 KB/109920 for c0901aa47578fba1a7f58734a4a0e3b2 in 277ms, sequenceid=254, compaction requested=false 2024-04-14T11:16:29,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:29,889 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:29,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-04-14T11:16:29,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:29,890 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-04-14T11:16:29,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/dab56de4ef2c4dbfa7f3d69119c56e32 is 50, key is test_row_0/A:col10/1713093389653/Put/seqid=0 2024-04-14T11:16:29,948 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/dab56de4ef2c4dbfa7f3d69119c56e32 2024-04-14T11:16:29,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:29,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:29,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/296996bf35664c14888bde44ccf3d3ad is 50, key is test_row_0/B:col10/1713093389653/Put/seqid=0 2024-04-14T11:16:30,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093449998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093449999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093450001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093450002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093450004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093450104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093450105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093450110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093450113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093450113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=75 2024-04-14T11:16:30,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093450309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093450311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093450317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093450319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093450323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,395 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/296996bf35664c14888bde44ccf3d3ad 2024-04-14T11:16:30,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/862b1ccd63d24ffab3d898d064c49a74 is 50, key is test_row_0/C:col10/1713093389653/Put/seqid=0 2024-04-14T11:16:30,451 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/862b1ccd63d24ffab3d898d064c49a74 2024-04-14T11:16:30,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/dab56de4ef2c4dbfa7f3d69119c56e32 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/dab56de4ef2c4dbfa7f3d69119c56e32 2024-04-14T11:16:30,484 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/dab56de4ef2c4dbfa7f3d69119c56e32, entries=150, sequenceid=276, filesize=12.0 K 2024-04-14T11:16:30,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/296996bf35664c14888bde44ccf3d3ad as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/296996bf35664c14888bde44ccf3d3ad 2024-04-14T11:16:30,501 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/296996bf35664c14888bde44ccf3d3ad, entries=150, sequenceid=276, filesize=12.0 K 2024-04-14T11:16:30,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/862b1ccd63d24ffab3d898d064c49a74 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/862b1ccd63d24ffab3d898d064c49a74 2024-04-14T11:16:30,512 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/862b1ccd63d24ffab3d898d064c49a74, entries=150, sequenceid=276, filesize=12.0 K 2024-04-14T11:16:30,517 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(3035): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for c0901aa47578fba1a7f58734a4a0e3b2 in 625ms, sequenceid=276, compaction requested=true 2024-04-14T11:16:30,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:30,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:30,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=76 2024-04-14T11:16:30,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=76 2024-04-14T11:16:30,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=76, resume processing ppid=75 2024-04-14T11:16:30,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=76, ppid=75, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4060 sec 2024-04-14T11:16:30,535 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=75, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees in 1.4260 sec 2024-04-14T11:16:30,615 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:30,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-04-14T11:16:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:30,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/b3762db97a03466ba358562a058873ce is 50, key is test_row_0/A:col10/1713093390613/Put/seqid=0 2024-04-14T11:16:30,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093450646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093450647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093450647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093450648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093450650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/b3762db97a03466ba358562a058873ce 2024-04-14T11:16:30,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/d03d9de8bde64b5c9c591980dfaec1d5 is 50, key is test_row_0/B:col10/1713093390613/Put/seqid=0 2024-04-14T11:16:30,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/d03d9de8bde64b5c9c591980dfaec1d5 2024-04-14T11:16:30,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/f9dc4b5855c84031898e39f1adef88a1 is 50, key is test_row_0/C:col10/1713093390613/Put/seqid=0 2024-04-14T11:16:30,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093450753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093450753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093450753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093450755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093450755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:30,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/f9dc4b5855c84031898e39f1adef88a1 2024-04-14T11:16:30,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/b3762db97a03466ba358562a058873ce as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/b3762db97a03466ba358562a058873ce 2024-04-14T11:16:30,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/b3762db97a03466ba358562a058873ce, entries=150, sequenceid=295, filesize=12.0 K 2024-04-14T11:16:30,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/d03d9de8bde64b5c9c591980dfaec1d5 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/d03d9de8bde64b5c9c591980dfaec1d5 2024-04-14T11:16:30,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/d03d9de8bde64b5c9c591980dfaec1d5, entries=150, sequenceid=295, filesize=12.0 K 2024-04-14T11:16:30,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/f9dc4b5855c84031898e39f1adef88a1 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/f9dc4b5855c84031898e39f1adef88a1 2024-04-14T11:16:30,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/f9dc4b5855c84031898e39f1adef88a1, entries=150, sequenceid=295, filesize=12.0 K 2024-04-14T11:16:30,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for c0901aa47578fba1a7f58734a4a0e3b2 in 233ms, sequenceid=295, compaction requested=true 2024-04-14T11:16:30,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:30,848 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:30,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:30,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:30,849 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:30,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:30,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:30,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:30,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:30,854 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:30,854 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/A is initiating minor compaction (all files) 2024-04-14T11:16:30,854 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/A in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:30,855 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/c9264ba28c884b689b4f63fe2935dd1b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/df1526798ebd457b9b43f3a36f833639, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/dab56de4ef2c4dbfa7f3d69119c56e32, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/b3762db97a03466ba358562a058873ce] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=55.2 K 2024-04-14T11:16:30,855 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:30,855 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/B is initiating minor compaction (all files) 2024-04-14T11:16:30,855 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/B in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:30,856 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f87225c8929942abb75fd4010c2b2ad7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/840b2decca8946b1a065decae005847e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/296996bf35664c14888bde44ccf3d3ad, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/d03d9de8bde64b5c9c591980dfaec1d5] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=48.2 K 2024-04-14T11:16:30,857 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c9264ba28c884b689b4f63fe2935dd1b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1713093388735 2024-04-14T11:16:30,857 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting f87225c8929942abb75fd4010c2b2ad7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1713093388735 2024-04-14T11:16:30,858 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting df1526798ebd457b9b43f3a36f833639, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1713093388879 2024-04-14T11:16:30,858 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 840b2decca8946b1a065decae005847e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1713093388879 2024-04-14T11:16:30,859 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting dab56de4ef2c4dbfa7f3d69119c56e32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1713093389635 2024-04-14T11:16:30,859 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 296996bf35664c14888bde44ccf3d3ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1713093389635 2024-04-14T11:16:30,860 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting d03d9de8bde64b5c9c591980dfaec1d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1713093389997 2024-04-14T11:16:30,860 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting b3762db97a03466ba358562a058873ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1713093389997 2024-04-14T11:16:30,876 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#A#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:30,876 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/8d692a9e7ce74e659330bf7dfa416df6 is 50, key is test_row_0/A:col10/1713093390613/Put/seqid=0 2024-04-14T11:16:30,888 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#B#compaction#275 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:30,888 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/2c195e64efb14cff9ca4a7210ccb6821 is 50, key is test_row_0/B:col10/1713093390613/Put/seqid=0 2024-04-14T11:16:30,952 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/2c195e64efb14cff9ca4a7210ccb6821 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/2c195e64efb14cff9ca4a7210ccb6821 2024-04-14T11:16:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:30,962 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:30,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-04-14T11:16:30,964 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/B of c0901aa47578fba1a7f58734a4a0e3b2 into 2c195e64efb14cff9ca4a7210ccb6821(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:30,964 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:30,964 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/B, priority=12, startTime=1713093390849; duration=0sec 2024-04-14T11:16:30,965 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:30,965 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:B 2024-04-14T11:16:30,965 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:30,969 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:30,969 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/C is initiating minor compaction (all files) 2024-04-14T11:16:30,969 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/C in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:30,969 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a0c6e3ef70b54436b24f95419251f811, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/ccacfd255932421bbec7550f0a788503, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/862b1ccd63d24ffab3d898d064c49a74, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/f9dc4b5855c84031898e39f1adef88a1] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=48.2 K 2024-04-14T11:16:30,971 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting a0c6e3ef70b54436b24f95419251f811, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1713093388735 2024-04-14T11:16:30,971 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting ccacfd255932421bbec7550f0a788503, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1713093388879 2024-04-14T11:16:30,972 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 862b1ccd63d24ffab3d898d064c49a74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1713093389635 2024-04-14T11:16:30,973 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting f9dc4b5855c84031898e39f1adef88a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1713093389997 2024-04-14T11:16:30,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/cf95a32f5a9c458bb70610814c5d4ec9 is 50, key is test_row_0/A:col10/1713093390960/Put/seqid=0 2024-04-14T11:16:30,995 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#C#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:30,995 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/0c61e200cba44f9fb8a1690373f06207 is 50, key is test_row_0/C:col10/1713093390613/Put/seqid=0 2024-04-14T11:16:31,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093450998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093450997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093450999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093450999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093451001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/cf95a32f5a9c458bb70610814c5d4ec9 2024-04-14T11:16:31,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/b4f8c2822ba24305b9629c2f211fb0e8 is 50, key is test_row_0/B:col10/1713093390960/Put/seqid=0 2024-04-14T11:16:31,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/b4f8c2822ba24305b9629c2f211fb0e8 2024-04-14T11:16:31,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093451106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093451106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093451106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093451106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093451108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/7441c59e40af4bd890a5476a1475758c is 50, key is test_row_0/C:col10/1713093390960/Put/seqid=0 2024-04-14T11:16:31,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=75 2024-04-14T11:16:31,221 INFO [Thread-1198 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 75 completed 2024-04-14T11:16:31,223 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:31,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=77, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees 2024-04-14T11:16:31,231 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=77, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=77 2024-04-14T11:16:31,232 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=77, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:31,233 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:31,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093451316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093451316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093451317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093451317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093451318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=77 2024-04-14T11:16:31,357 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/8d692a9e7ce74e659330bf7dfa416df6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/8d692a9e7ce74e659330bf7dfa416df6 2024-04-14T11:16:31,374 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/A of c0901aa47578fba1a7f58734a4a0e3b2 into 8d692a9e7ce74e659330bf7dfa416df6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:31,375 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:31,375 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/A, priority=12, startTime=1713093390848; duration=0sec 2024-04-14T11:16:31,375 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:31,375 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:A 2024-04-14T11:16:31,387 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2024-04-14T11:16:31,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:31,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:31,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:31,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(53): pid=78 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:31,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:31,485 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/0c61e200cba44f9fb8a1690373f06207 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/0c61e200cba44f9fb8a1690373f06207 2024-04-14T11:16:31,496 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/C of c0901aa47578fba1a7f58734a4a0e3b2 into 0c61e200cba44f9fb8a1690373f06207(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:31,496 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:31,496 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/C, priority=12, startTime=1713093390850; duration=0sec 2024-04-14T11:16:31,496 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:31,497 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:C 2024-04-14T11:16:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=77 2024-04-14T11:16:31,541 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2024-04-14T11:16:31,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:31,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:31,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:31,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(53): pid=78 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:31,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:31,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/7441c59e40af4bd890a5476a1475758c 2024-04-14T11:16:31,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/cf95a32f5a9c458bb70610814c5d4ec9 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/cf95a32f5a9c458bb70610814c5d4ec9 2024-04-14T11:16:31,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/cf95a32f5a9c458bb70610814c5d4ec9, entries=200, sequenceid=313, filesize=14.4 K 2024-04-14T11:16:31,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/b4f8c2822ba24305b9629c2f211fb0e8 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/b4f8c2822ba24305b9629c2f211fb0e8 2024-04-14T11:16:31,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093451621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/b4f8c2822ba24305b9629c2f211fb0e8, entries=150, sequenceid=313, filesize=12.0 K 2024-04-14T11:16:31,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093451622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/7441c59e40af4bd890a5476a1475758c as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/7441c59e40af4bd890a5476a1475758c 2024-04-14T11:16:31,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093451623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093451623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093451623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/7441c59e40af4bd890a5476a1475758c, entries=150, sequenceid=313, filesize=12.0 K 2024-04-14T11:16:31,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for c0901aa47578fba1a7f58734a4a0e3b2 in 677ms, sequenceid=313, compaction requested=false 2024-04-14T11:16:31,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:31,696 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:31,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2024-04-14T11:16:31,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:31,697 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-04-14T11:16:31,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/eed8dd27410046158e6f8c7236436d28 is 50, key is test_row_0/A:col10/1713093390997/Put/seqid=0 2024-04-14T11:16:31,730 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/eed8dd27410046158e6f8c7236436d28 2024-04-14T11:16:31,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/e26d1bdd030b451381ee04da62cc0490 is 50, key is test_row_0/B:col10/1713093390997/Put/seqid=0 2024-04-14T11:16:31,766 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/e26d1bdd030b451381ee04da62cc0490 2024-04-14T11:16:31,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/a4043a3714c6404aa574e454122ff07f is 50, key is test_row_0/C:col10/1713093390997/Put/seqid=0 2024-04-14T11:16:31,801 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/a4043a3714c6404aa574e454122ff07f 2024-04-14T11:16:31,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/eed8dd27410046158e6f8c7236436d28 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/eed8dd27410046158e6f8c7236436d28 2024-04-14T11:16:31,819 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/eed8dd27410046158e6f8c7236436d28, entries=150, sequenceid=334, filesize=12.0 K 2024-04-14T11:16:31,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/e26d1bdd030b451381ee04da62cc0490 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/e26d1bdd030b451381ee04da62cc0490 2024-04-14T11:16:31,831 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/e26d1bdd030b451381ee04da62cc0490, entries=150, sequenceid=334, filesize=12.0 K 2024-04-14T11:16:31,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/a4043a3714c6404aa574e454122ff07f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a4043a3714c6404aa574e454122ff07f 2024-04-14T11:16:31,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=77 2024-04-14T11:16:31,841 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a4043a3714c6404aa574e454122ff07f, entries=150, sequenceid=334, filesize=12.0 K 2024-04-14T11:16:31,842 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(3035): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=0 B/0 for c0901aa47578fba1a7f58734a4a0e3b2 in 144ms, sequenceid=334, compaction requested=true 2024-04-14T11:16:31,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:31,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:31,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=78 2024-04-14T11:16:31,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=78 2024-04-14T11:16:31,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=78, resume processing ppid=77 2024-04-14T11:16:31,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=78, ppid=77, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 610 msec 2024-04-14T11:16:31,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=77, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees in 628 msec 2024-04-14T11:16:32,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:32,144 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:32,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-04-14T11:16:32,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/fcb4249c23454cd4a652cba4dcddaa30 is 50, key is test_row_0/A:col10/1713093392139/Put/seqid=0 2024-04-14T11:16:32,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093452174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/fcb4249c23454cd4a652cba4dcddaa30 2024-04-14T11:16:32,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093452178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093452182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093452182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093452182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/cd1da2f729a044af91ad3f50f63ecd0b is 50, key is test_row_0/B:col10/1713093392139/Put/seqid=0 2024-04-14T11:16:32,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093452283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093452289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093452289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093452290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093452298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=77 2024-04-14T11:16:32,337 INFO [Thread-1198 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 77 completed 2024-04-14T11:16:32,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=79, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees 2024-04-14T11:16:32,348 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=79, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:32,349 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=79, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:32,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:32,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2024-04-14T11:16:32,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2024-04-14T11:16:32,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093452488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093452492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093452494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093452494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093452502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,503 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-04-14T11:16:32,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:32,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:32,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:32,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(53): pid=80 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:32,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:32,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:32,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/cd1da2f729a044af91ad3f50f63ecd0b 2024-04-14T11:16:32,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/2606205d9b4b45f898436804ea8de3e6 is 50, key is test_row_0/C:col10/1713093392139/Put/seqid=0 2024-04-14T11:16:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2024-04-14T11:16:32,657 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-04-14T11:16:32,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:32,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:32,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:32,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(53): pid=80 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:32,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:32,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093452792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093452798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093452800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093452801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:32,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093452807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,812 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-04-14T11:16:32,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:32,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:32,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:32,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(53): pid=80 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:32,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:32,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2024-04-14T11:16:32,972 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:32,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-04-14T11:16:32,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:32,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:32,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:32,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(53): pid=80 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:32,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:33,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/2606205d9b4b45f898436804ea8de3e6 2024-04-14T11:16:33,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/fcb4249c23454cd4a652cba4dcddaa30 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/fcb4249c23454cd4a652cba4dcddaa30 2024-04-14T11:16:33,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/fcb4249c23454cd4a652cba4dcddaa30, entries=150, sequenceid=347, filesize=12.0 K 2024-04-14T11:16:33,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/cd1da2f729a044af91ad3f50f63ecd0b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/cd1da2f729a044af91ad3f50f63ecd0b 2024-04-14T11:16:33,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/cd1da2f729a044af91ad3f50f63ecd0b, entries=150, sequenceid=347, filesize=12.0 K 2024-04-14T11:16:33,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/2606205d9b4b45f898436804ea8de3e6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2606205d9b4b45f898436804ea8de3e6 2024-04-14T11:16:33,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2606205d9b4b45f898436804ea8de3e6, entries=150, sequenceid=347, filesize=12.0 K 2024-04-14T11:16:33,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c0901aa47578fba1a7f58734a4a0e3b2 in 939ms, sequenceid=347, compaction requested=true 2024-04-14T11:16:33,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:33,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:33,083 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:33,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:33,083 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:33,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:33,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:33,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:33,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:33,087 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:33,087 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/B is initiating minor compaction (all files) 2024-04-14T11:16:33,087 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/B in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:33,088 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/2c195e64efb14cff9ca4a7210ccb6821, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/b4f8c2822ba24305b9629c2f211fb0e8, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/e26d1bdd030b451381ee04da62cc0490, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/cd1da2f729a044af91ad3f50f63ecd0b] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=48.7 K 2024-04-14T11:16:33,090 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52258 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:33,090 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/A is initiating minor compaction (all files) 2024-04-14T11:16:33,090 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/A in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:33,090 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/8d692a9e7ce74e659330bf7dfa416df6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/cf95a32f5a9c458bb70610814c5d4ec9, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/eed8dd27410046158e6f8c7236436d28, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/fcb4249c23454cd4a652cba4dcddaa30] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=51.0 K 2024-04-14T11:16:33,092 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 2c195e64efb14cff9ca4a7210ccb6821, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1713093389997 2024-04-14T11:16:33,094 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8d692a9e7ce74e659330bf7dfa416df6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1713093389997 2024-04-14T11:16:33,095 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting b4f8c2822ba24305b9629c2f211fb0e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1713093390642 2024-04-14T11:16:33,095 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting cf95a32f5a9c458bb70610814c5d4ec9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1713093390642 2024-04-14T11:16:33,096 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting e26d1bdd030b451381ee04da62cc0490, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1713093390993 2024-04-14T11:16:33,096 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting eed8dd27410046158e6f8c7236436d28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1713093390993 2024-04-14T11:16:33,097 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting cd1da2f729a044af91ad3f50f63ecd0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1713093392137 2024-04-14T11:16:33,097 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting fcb4249c23454cd4a652cba4dcddaa30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1713093392137 2024-04-14T11:16:33,118 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#A#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:33,119 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/bad60be8bfce4955aa80d3a2184b04fd is 50, key is test_row_0/A:col10/1713093392139/Put/seqid=0 2024-04-14T11:16:33,125 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:33,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-04-14T11:16:33,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:33,126 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-04-14T11:16:33,130 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#B#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:33,131 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/1534a7d91d9c4352a012a263005b1fcc is 50, key is test_row_0/B:col10/1713093392139/Put/seqid=0 2024-04-14T11:16:33,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/2a52e0300bb84abe998e8013e3496141 is 50, key is test_row_0/A:col10/1713093392175/Put/seqid=0 2024-04-14T11:16:33,199 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/bad60be8bfce4955aa80d3a2184b04fd as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/bad60be8bfce4955aa80d3a2184b04fd 2024-04-14T11:16:33,208 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/A of c0901aa47578fba1a7f58734a4a0e3b2 into bad60be8bfce4955aa80d3a2184b04fd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:33,208 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:33,208 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/A, priority=12, startTime=1713093393083; duration=0sec 2024-04-14T11:16:33,208 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:33,208 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:A 2024-04-14T11:16:33,208 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:33,215 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:33,215 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/C is initiating minor compaction (all files) 2024-04-14T11:16:33,215 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/C in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:33,216 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/0c61e200cba44f9fb8a1690373f06207, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/7441c59e40af4bd890a5476a1475758c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a4043a3714c6404aa574e454122ff07f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2606205d9b4b45f898436804ea8de3e6] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=48.7 K 2024-04-14T11:16:33,217 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 0c61e200cba44f9fb8a1690373f06207, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1713093389997 2024-04-14T11:16:33,218 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7441c59e40af4bd890a5476a1475758c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1713093390642 2024-04-14T11:16:33,219 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting a4043a3714c6404aa574e454122ff07f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1713093390993 2024-04-14T11:16:33,219 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2606205d9b4b45f898436804ea8de3e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1713093392137 2024-04-14T11:16:33,226 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/2a52e0300bb84abe998e8013e3496141 2024-04-14T11:16:33,236 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/1534a7d91d9c4352a012a263005b1fcc as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/1534a7d91d9c4352a012a263005b1fcc 2024-04-14T11:16:33,249 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/B of c0901aa47578fba1a7f58734a4a0e3b2 into 1534a7d91d9c4352a012a263005b1fcc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:33,249 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:33,249 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/B, priority=12, startTime=1713093393083; duration=0sec 2024-04-14T11:16:33,249 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:33,249 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:B 2024-04-14T11:16:33,250 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#C#compaction#289 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:33,251 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/287114d8854d491880f1c36eeccc8f1e is 50, key is test_row_0/C:col10/1713093392139/Put/seqid=0 2024-04-14T11:16:33,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/bd30a71ec07b4835be75d98ba18a8d8e is 50, key is test_row_0/B:col10/1713093392175/Put/seqid=0 2024-04-14T11:16:33,280 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/287114d8854d491880f1c36eeccc8f1e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/287114d8854d491880f1c36eeccc8f1e 2024-04-14T11:16:33,289 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/C of c0901aa47578fba1a7f58734a4a0e3b2 into 287114d8854d491880f1c36eeccc8f1e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:33,289 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:33,289 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/C, priority=12, startTime=1713093393084; duration=0sec 2024-04-14T11:16:33,289 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:33,289 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:C 2024-04-14T11:16:33,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:33,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:33,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:33,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093453354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:33,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:33,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093453354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:33,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093453364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:33,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093453364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:33,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:33,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093453365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:33,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2024-04-14T11:16:33,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:33,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093453469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:33,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:33,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093453469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:33,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:33,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093453470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:33,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093453471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:33,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093453471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:33,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093453673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:33,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093453675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:33,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093453675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:33,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:33,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:33,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093453678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:33,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093453678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:33,692 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/bd30a71ec07b4835be75d98ba18a8d8e 2024-04-14T11:16:33,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/4b1fc854b0224414a2483d8722e3f793 is 50, key is test_row_0/C:col10/1713093392175/Put/seqid=0 2024-04-14T11:16:33,712 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/4b1fc854b0224414a2483d8722e3f793 2024-04-14T11:16:33,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/2a52e0300bb84abe998e8013e3496141 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2a52e0300bb84abe998e8013e3496141 2024-04-14T11:16:33,724 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2a52e0300bb84abe998e8013e3496141, entries=150, sequenceid=370, filesize=12.0 K 2024-04-14T11:16:33,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/bd30a71ec07b4835be75d98ba18a8d8e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/bd30a71ec07b4835be75d98ba18a8d8e 2024-04-14T11:16:33,730 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/bd30a71ec07b4835be75d98ba18a8d8e, entries=150, sequenceid=370, filesize=12.0 K 2024-04-14T11:16:33,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/4b1fc854b0224414a2483d8722e3f793 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/4b1fc854b0224414a2483d8722e3f793 2024-04-14T11:16:33,737 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/4b1fc854b0224414a2483d8722e3f793, entries=150, sequenceid=370, filesize=12.0 K 2024-04-14T11:16:33,737 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(3035): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for c0901aa47578fba1a7f58734a4a0e3b2 in 611ms, sequenceid=370, compaction requested=false 2024-04-14T11:16:33,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:33,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:33,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=80 2024-04-14T11:16:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=80 2024-04-14T11:16:33,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=80, resume processing ppid=79 2024-04-14T11:16:33,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=80, ppid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3900 sec 2024-04-14T11:16:33,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees in 1.3980 sec 2024-04-14T11:16:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:33,979 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:33,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-04-14T11:16:33,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/7dbc5dd0a638447db170696dcc3f9931 is 50, key is test_row_0/A:col10/1713093393319/Put/seqid=0 2024-04-14T11:16:34,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093454003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093454004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093454005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093454005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093454006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093454109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093454110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093454110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093454110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093454110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093454314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093454314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093454314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093454314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093454315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/7dbc5dd0a638447db170696dcc3f9931 2024-04-14T11:16:34,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/f73d48bd452f4fd0a7b3ce87de82a024 is 50, key is test_row_0/B:col10/1713093393319/Put/seqid=0 2024-04-14T11:16:34,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/f73d48bd452f4fd0a7b3ce87de82a024 2024-04-14T11:16:34,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/25cfb45c22d0412692fd1299adfb67fe is 50, key is test_row_0/C:col10/1713093393319/Put/seqid=0 2024-04-14T11:16:34,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/25cfb45c22d0412692fd1299adfb67fe 2024-04-14T11:16:34,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/7dbc5dd0a638447db170696dcc3f9931 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/7dbc5dd0a638447db170696dcc3f9931 2024-04-14T11:16:34,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/7dbc5dd0a638447db170696dcc3f9931, entries=200, sequenceid=390, filesize=14.4 K 2024-04-14T11:16:34,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/f73d48bd452f4fd0a7b3ce87de82a024 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f73d48bd452f4fd0a7b3ce87de82a024 2024-04-14T11:16:34,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f73d48bd452f4fd0a7b3ce87de82a024, entries=150, sequenceid=390, filesize=12.0 K 2024-04-14T11:16:34,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/25cfb45c22d0412692fd1299adfb67fe as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/25cfb45c22d0412692fd1299adfb67fe 2024-04-14T11:16:34,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/25cfb45c22d0412692fd1299adfb67fe, entries=150, sequenceid=390, filesize=12.0 K 2024-04-14T11:16:34,450 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for c0901aa47578fba1a7f58734a4a0e3b2 in 471ms, sequenceid=390, compaction requested=true 2024-04-14T11:16:34,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:34,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:34,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:34,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:34,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:34,451 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:34,451 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:34,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:34,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:34,452 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40110 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:34,452 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/A is initiating minor compaction (all files) 2024-04-14T11:16:34,452 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/A in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:34,452 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/bad60be8bfce4955aa80d3a2184b04fd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2a52e0300bb84abe998e8013e3496141, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/7dbc5dd0a638447db170696dcc3f9931] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=39.2 K 2024-04-14T11:16:34,452 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37670 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:34,453 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/B is initiating minor compaction (all files) 2024-04-14T11:16:34,453 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/B in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:34,453 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting bad60be8bfce4955aa80d3a2184b04fd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1713093392137 2024-04-14T11:16:34,453 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/1534a7d91d9c4352a012a263005b1fcc, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/bd30a71ec07b4835be75d98ba18a8d8e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f73d48bd452f4fd0a7b3ce87de82a024] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=36.8 K 2024-04-14T11:16:34,455 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2a52e0300bb84abe998e8013e3496141, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1713093392175 2024-04-14T11:16:34,455 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 1534a7d91d9c4352a012a263005b1fcc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1713093392137 2024-04-14T11:16:34,455 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7dbc5dd0a638447db170696dcc3f9931, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1713093393319 2024-04-14T11:16:34,455 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting bd30a71ec07b4835be75d98ba18a8d8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1713093392175 2024-04-14T11:16:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2024-04-14T11:16:34,457 INFO [Thread-1198 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 79 completed 2024-04-14T11:16:34,459 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting f73d48bd452f4fd0a7b3ce87de82a024, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1713093393319 2024-04-14T11:16:34,459 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:34,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=81, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees 2024-04-14T11:16:34,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2024-04-14T11:16:34,470 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#A#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:34,470 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#B#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:34,470 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/099e610cccb8461486e1a927665cd335 is 50, key is test_row_0/A:col10/1713093393319/Put/seqid=0 2024-04-14T11:16:34,470 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/8dd7b20164e94073b31037f685194213 is 50, key is test_row_0/B:col10/1713093393319/Put/seqid=0 2024-04-14T11:16:34,471 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=81, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:34,473 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=81, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:34,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:34,495 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/8dd7b20164e94073b31037f685194213 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/8dd7b20164e94073b31037f685194213 2024-04-14T11:16:34,504 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/B of c0901aa47578fba1a7f58734a4a0e3b2 into 8dd7b20164e94073b31037f685194213(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:34,504 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:34,504 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/B, priority=13, startTime=1713093394451; duration=0sec 2024-04-14T11:16:34,504 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:34,504 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:B 2024-04-14T11:16:34,504 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:34,506 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37670 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:34,506 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/C is initiating minor compaction (all files) 2024-04-14T11:16:34,506 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/C in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:34,506 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/287114d8854d491880f1c36eeccc8f1e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/4b1fc854b0224414a2483d8722e3f793, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/25cfb45c22d0412692fd1299adfb67fe] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=36.8 K 2024-04-14T11:16:34,507 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 287114d8854d491880f1c36eeccc8f1e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1713093392137 2024-04-14T11:16:34,507 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 4b1fc854b0224414a2483d8722e3f793, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1713093392175 2024-04-14T11:16:34,508 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 25cfb45c22d0412692fd1299adfb67fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1713093393319 2024-04-14T11:16:34,517 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#C#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:34,518 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/2568de2b4f8a4c298328e27e0b5a0525 is 50, key is test_row_0/C:col10/1713093393319/Put/seqid=0 2024-04-14T11:16:34,542 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/2568de2b4f8a4c298328e27e0b5a0525 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2568de2b4f8a4c298328e27e0b5a0525 2024-04-14T11:16:34,549 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/C of c0901aa47578fba1a7f58734a4a0e3b2 into 2568de2b4f8a4c298328e27e0b5a0525(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:34,549 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:34,549 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/C, priority=13, startTime=1713093394451; duration=0sec 2024-04-14T11:16:34,549 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:34,549 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:C 2024-04-14T11:16:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2024-04-14T11:16:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:34,618 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:34,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-04-14T11:16:34,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/a913205557ff425b8eb03117d768454e is 50, key is test_row_0/A:col10/1713093394004/Put/seqid=0 2024-04-14T11:16:34,625 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-04-14T11:16:34,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:34,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:34,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:34,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(53): pid=82 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:34,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:34,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/a913205557ff425b8eb03117d768454e 2024-04-14T11:16:34,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093454633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093454636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093454636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093454638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093454638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/1e3c7aa35384457da9f4c1e8fb6c8760 is 50, key is test_row_0/B:col10/1713093394004/Put/seqid=0 2024-04-14T11:16:34,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/1e3c7aa35384457da9f4c1e8fb6c8760 2024-04-14T11:16:34,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/743702e628754c8aa1f449fdd6bdd5dc is 50, key is test_row_0/C:col10/1713093394004/Put/seqid=0 2024-04-14T11:16:34,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093454739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093454740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093454746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093454746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093454746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2024-04-14T11:16:34,784 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-04-14T11:16:34,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:34,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:34,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:34,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(53): pid=82 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:34,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:34,887 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/099e610cccb8461486e1a927665cd335 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/099e610cccb8461486e1a927665cd335 2024-04-14T11:16:34,898 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/A of c0901aa47578fba1a7f58734a4a0e3b2 into 099e610cccb8461486e1a927665cd335(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:34,898 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:34,898 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/A, priority=13, startTime=1713093394450; duration=0sec 2024-04-14T11:16:34,898 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:34,898 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:A 2024-04-14T11:16:34,938 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-04-14T11:16:34,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:34,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:34,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:34,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(53): pid=82 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:34,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:34,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093454942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093454943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093454948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093454949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:34,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093454949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2024-04-14T11:16:35,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/743702e628754c8aa1f449fdd6bdd5dc 2024-04-14T11:16:35,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/a913205557ff425b8eb03117d768454e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/a913205557ff425b8eb03117d768454e 2024-04-14T11:16:35,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/a913205557ff425b8eb03117d768454e, entries=150, sequenceid=412, filesize=12.0 K 2024-04-14T11:16:35,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/1e3c7aa35384457da9f4c1e8fb6c8760 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/1e3c7aa35384457da9f4c1e8fb6c8760 2024-04-14T11:16:35,091 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-04-14T11:16:35,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:35,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:35,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:35,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(53): pid=82 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:35,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:35,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:35,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/1e3c7aa35384457da9f4c1e8fb6c8760, entries=150, sequenceid=412, filesize=12.0 K 2024-04-14T11:16:35,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/743702e628754c8aa1f449fdd6bdd5dc as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/743702e628754c8aa1f449fdd6bdd5dc 2024-04-14T11:16:35,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/743702e628754c8aa1f449fdd6bdd5dc, entries=150, sequenceid=412, filesize=12.0 K 2024-04-14T11:16:35,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for c0901aa47578fba1a7f58734a4a0e3b2 in 483ms, sequenceid=412, compaction requested=false 2024-04-14T11:16:35,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:35,244 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-04-14T11:16:35,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:35,245 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-04-14T11:16:35,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:35,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:35,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/5c0cbbaa750f46318170d40bde0e4c5c is 50, key is test_row_0/A:col10/1713093394636/Put/seqid=0 2024-04-14T11:16:35,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093455258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093455260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093455261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093455261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093455262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093455363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093455364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093455364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093455365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093455365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2024-04-14T11:16:35,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093455567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093455567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093455567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093455568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093455568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,656 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/5c0cbbaa750f46318170d40bde0e4c5c 2024-04-14T11:16:35,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/c0366f52f3e245eda187348a72a77134 is 50, key is test_row_0/B:col10/1713093394636/Put/seqid=0 2024-04-14T11:16:35,670 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/c0366f52f3e245eda187348a72a77134 2024-04-14T11:16:35,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/c20eecdb953542ce8a4e64db7c2bdda3 is 50, key is test_row_0/C:col10/1713093394636/Put/seqid=0 2024-04-14T11:16:35,684 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/c20eecdb953542ce8a4e64db7c2bdda3 2024-04-14T11:16:35,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/5c0cbbaa750f46318170d40bde0e4c5c as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5c0cbbaa750f46318170d40bde0e4c5c 2024-04-14T11:16:35,695 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5c0cbbaa750f46318170d40bde0e4c5c, entries=150, sequenceid=430, filesize=12.0 K 2024-04-14T11:16:35,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/c0366f52f3e245eda187348a72a77134 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/c0366f52f3e245eda187348a72a77134 2024-04-14T11:16:35,702 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/c0366f52f3e245eda187348a72a77134, entries=150, sequenceid=430, filesize=12.0 K 2024-04-14T11:16:35,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/c20eecdb953542ce8a4e64db7c2bdda3 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c20eecdb953542ce8a4e64db7c2bdda3 2024-04-14T11:16:35,713 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c20eecdb953542ce8a4e64db7c2bdda3, entries=150, sequenceid=430, filesize=12.0 K 2024-04-14T11:16:35,714 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(3035): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for c0901aa47578fba1a7f58734a4a0e3b2 in 469ms, sequenceid=430, compaction requested=true 2024-04-14T11:16:35,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:35,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:35,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=82 2024-04-14T11:16:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=82 2024-04-14T11:16:35,719 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=82, resume processing ppid=81 2024-04-14T11:16:35,719 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=82, ppid=81, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2430 sec 2024-04-14T11:16:35,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=81, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees in 1.2610 sec 2024-04-14T11:16:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:35,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-04-14T11:16:35,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/5085c5eaab55467e9cd93d9875b6f979 is 50, key is test_row_0/A:col10/1713093395871/Put/seqid=0 2024-04-14T11:16:35,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093455883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093455885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093455886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093455891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093455891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093455992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093455992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093455992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093455995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:35,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093455995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:36,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093456195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:36,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093456195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:36,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:36,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093456195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:36,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:36,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093456197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:36,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093456199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:36,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/5085c5eaab55467e9cd93d9875b6f979 2024-04-14T11:16:36,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/ca387b1fb9dd44efba1ffae40531b3b7 is 50, key is test_row_0/B:col10/1713093395871/Put/seqid=0 2024-04-14T11:16:36,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/ca387b1fb9dd44efba1ffae40531b3b7 2024-04-14T11:16:36,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/527fe695f8d44c82baae5b17f54e9386 is 50, key is test_row_0/C:col10/1713093395871/Put/seqid=0 2024-04-14T11:16:36,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093456497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:36,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093456499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:36,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093456500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:36,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093456501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:36,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093456502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2024-04-14T11:16:36,568 INFO [Thread-1198 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 81 completed 2024-04-14T11:16:36,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees 2024-04-14T11:16:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=83 2024-04-14T11:16:36,573 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:36,574 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=83, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:36,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=83 2024-04-14T11:16:36,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/527fe695f8d44c82baae5b17f54e9386 2024-04-14T11:16:36,726 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:36,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/5085c5eaab55467e9cd93d9875b6f979 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5085c5eaab55467e9cd93d9875b6f979 2024-04-14T11:16:36,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-04-14T11:16:36,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:36,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:36,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:36,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(53): pid=84 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:36,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:36,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5085c5eaab55467e9cd93d9875b6f979, entries=150, sequenceid=451, filesize=12.0 K 2024-04-14T11:16:36,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/ca387b1fb9dd44efba1ffae40531b3b7 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/ca387b1fb9dd44efba1ffae40531b3b7 2024-04-14T11:16:36,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/ca387b1fb9dd44efba1ffae40531b3b7, entries=150, sequenceid=451, filesize=12.0 K 2024-04-14T11:16:36,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/527fe695f8d44c82baae5b17f54e9386 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/527fe695f8d44c82baae5b17f54e9386 2024-04-14T11:16:36,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/527fe695f8d44c82baae5b17f54e9386, entries=150, sequenceid=451, filesize=12.0 K 2024-04-14T11:16:36,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for c0901aa47578fba1a7f58734a4a0e3b2 in 877ms, sequenceid=451, compaction requested=true 2024-04-14T11:16:36,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:36,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:36,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:36,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:36,750 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:36,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:36,750 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:36,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:36,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:36,752 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:36,752 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/A is initiating minor compaction (all files) 2024-04-14T11:16:36,752 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/A in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:36,753 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/099e610cccb8461486e1a927665cd335, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/a913205557ff425b8eb03117d768454e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5c0cbbaa750f46318170d40bde0e4c5c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5085c5eaab55467e9cd93d9875b6f979] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=48.9 K 2024-04-14T11:16:36,753 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 099e610cccb8461486e1a927665cd335, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1713093393319 2024-04-14T11:16:36,754 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting a913205557ff425b8eb03117d768454e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1713093394002 2024-04-14T11:16:36,754 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5c0cbbaa750f46318170d40bde0e4c5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1713093394630 2024-04-14T11:16:36,754 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:36,755 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/B is initiating minor compaction (all files) 2024-04-14T11:16:36,755 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/B in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:36,755 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/8dd7b20164e94073b31037f685194213, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/1e3c7aa35384457da9f4c1e8fb6c8760, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/c0366f52f3e245eda187348a72a77134, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/ca387b1fb9dd44efba1ffae40531b3b7] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=48.9 K 2024-04-14T11:16:36,755 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5085c5eaab55467e9cd93d9875b6f979, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1713093395260 2024-04-14T11:16:36,755 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 8dd7b20164e94073b31037f685194213, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1713093393319 2024-04-14T11:16:36,756 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 1e3c7aa35384457da9f4c1e8fb6c8760, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1713093394002 2024-04-14T11:16:36,756 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting c0366f52f3e245eda187348a72a77134, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1713093394630 2024-04-14T11:16:36,757 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting ca387b1fb9dd44efba1ffae40531b3b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1713093395260 2024-04-14T11:16:36,766 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#A#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:36,767 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/9e8292ff3838453faaad98f977320699 is 50, key is test_row_0/A:col10/1713093395871/Put/seqid=0 2024-04-14T11:16:36,767 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#B#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:36,768 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/e37b04c2cbdd40c4ad5555a9f649149a is 50, key is test_row_0/B:col10/1713093395871/Put/seqid=0 2024-04-14T11:16:36,781 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/9e8292ff3838453faaad98f977320699 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/9e8292ff3838453faaad98f977320699 2024-04-14T11:16:36,787 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/A of c0901aa47578fba1a7f58734a4a0e3b2 into 9e8292ff3838453faaad98f977320699(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:36,787 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:36,787 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/A, priority=12, startTime=1713093396750; duration=0sec 2024-04-14T11:16:36,788 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:36,788 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:A 2024-04-14T11:16:36,788 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:36,790 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:36,790 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/C is initiating minor compaction (all files) 2024-04-14T11:16:36,790 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/C in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:36,790 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2568de2b4f8a4c298328e27e0b5a0525, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/743702e628754c8aa1f449fdd6bdd5dc, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c20eecdb953542ce8a4e64db7c2bdda3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/527fe695f8d44c82baae5b17f54e9386] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=48.9 K 2024-04-14T11:16:36,791 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2568de2b4f8a4c298328e27e0b5a0525, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1713093393319 2024-04-14T11:16:36,791 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 743702e628754c8aa1f449fdd6bdd5dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1713093394002 2024-04-14T11:16:36,792 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c20eecdb953542ce8a4e64db7c2bdda3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1713093394630 2024-04-14T11:16:36,792 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 527fe695f8d44c82baae5b17f54e9386, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1713093395260 2024-04-14T11:16:36,801 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#C#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:36,801 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/dcbe2fe3dcc94cbfb36405844f1f6bea is 50, key is test_row_0/C:col10/1713093395871/Put/seqid=0 2024-04-14T11:16:36,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=83 2024-04-14T11:16:36,879 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:36,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-04-14T11:16:36,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:36,880 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-04-14T11:16:36,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/e18f709ce3c940c7b3bb4cc9e59e3ffa is 50, key is test_row_0/A:col10/1713093395888/Put/seqid=0 2024-04-14T11:16:37,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:37,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:37,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093457019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093457022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093457023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093457024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093457024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093457125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093457125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093457127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093457127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093457127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=83 2024-04-14T11:16:37,182 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/e37b04c2cbdd40c4ad5555a9f649149a as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/e37b04c2cbdd40c4ad5555a9f649149a 2024-04-14T11:16:37,189 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/B of c0901aa47578fba1a7f58734a4a0e3b2 into e37b04c2cbdd40c4ad5555a9f649149a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:37,189 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:37,189 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/B, priority=12, startTime=1713093396750; duration=0sec 2024-04-14T11:16:37,189 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:37,189 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:B 2024-04-14T11:16:37,214 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/dcbe2fe3dcc94cbfb36405844f1f6bea as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/dcbe2fe3dcc94cbfb36405844f1f6bea 2024-04-14T11:16:37,221 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/C of c0901aa47578fba1a7f58734a4a0e3b2 into dcbe2fe3dcc94cbfb36405844f1f6bea(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:37,221 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:37,221 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/C, priority=12, startTime=1713093396750; duration=0sec 2024-04-14T11:16:37,221 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:37,221 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:C 2024-04-14T11:16:37,290 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/e18f709ce3c940c7b3bb4cc9e59e3ffa 2024-04-14T11:16:37,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/402b9cb8178b46e7935beef58047f2f2 is 50, key is test_row_0/B:col10/1713093395888/Put/seqid=0 2024-04-14T11:16:37,304 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/402b9cb8178b46e7935beef58047f2f2 2024-04-14T11:16:37,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/736b854f227f40159ff1c8876e7a3e37 is 50, key is test_row_0/C:col10/1713093395888/Put/seqid=0 2024-04-14T11:16:37,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093457327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093457328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093457330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093457330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093457331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,356 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/736b854f227f40159ff1c8876e7a3e37 2024-04-14T11:16:37,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/e18f709ce3c940c7b3bb4cc9e59e3ffa as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/e18f709ce3c940c7b3bb4cc9e59e3ffa 2024-04-14T11:16:37,371 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/e18f709ce3c940c7b3bb4cc9e59e3ffa, entries=150, sequenceid=468, filesize=12.0 K 2024-04-14T11:16:37,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/402b9cb8178b46e7935beef58047f2f2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/402b9cb8178b46e7935beef58047f2f2 2024-04-14T11:16:37,380 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/402b9cb8178b46e7935beef58047f2f2, entries=150, sequenceid=468, filesize=12.0 K 2024-04-14T11:16:37,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/736b854f227f40159ff1c8876e7a3e37 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/736b854f227f40159ff1c8876e7a3e37 2024-04-14T11:16:37,387 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/736b854f227f40159ff1c8876e7a3e37, entries=150, sequenceid=468, filesize=12.0 K 2024-04-14T11:16:37,388 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(3035): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for c0901aa47578fba1a7f58734a4a0e3b2 in 508ms, sequenceid=468, compaction requested=false 2024-04-14T11:16:37,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:37,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:37,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=84 2024-04-14T11:16:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=84 2024-04-14T11:16:37,392 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=84, resume processing ppid=83 2024-04-14T11:16:37,392 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=84, ppid=83, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 815 msec 2024-04-14T11:16:37,396 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=83, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees in 822 msec 2024-04-14T11:16:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:37,633 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:37,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-04-14T11:16:37,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/71e1e4ed9c1a4de797a22b50e1e2f4a6 is 50, key is test_row_0/A:col10/1713093397022/Put/seqid=0 2024-04-14T11:16:37,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093457642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093457643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093457644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093457644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093457645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=83 2024-04-14T11:16:37,678 INFO [Thread-1198 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 83 completed 2024-04-14T11:16:37,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:37,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees 2024-04-14T11:16:37,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=85 2024-04-14T11:16:37,683 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:37,683 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=85, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:37,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:37,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093457747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093457747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093457748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093457748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093457748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=85 2024-04-14T11:16:37,835 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-04-14T11:16:37,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:37,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:37,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:37,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(53): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:37,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:37,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093457949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093457950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093457951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093457951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093457951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=85 2024-04-14T11:16:37,988 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:37,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-04-14T11:16:37,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:37,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:37,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:37,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(53): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:37,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:38,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/71e1e4ed9c1a4de797a22b50e1e2f4a6 2024-04-14T11:16:38,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/13ff14b732e74380a329a2d11d5fe685 is 50, key is test_row_0/B:col10/1713093397022/Put/seqid=0 2024-04-14T11:16:38,141 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:38,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-04-14T11:16:38,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:38,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:38,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:38,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(53): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:38,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:38,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:38,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:38,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093458251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:38,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:38,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093458255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:38,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:38,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093458255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:38,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:38,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093458255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:38,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:38,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093458255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:38,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=85 2024-04-14T11:16:38,295 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:38,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-04-14T11:16:38,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:38,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:38,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:38,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(53): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:38,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:38,448 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:38,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-04-14T11:16:38,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:38,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:38,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:38,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(53): pid=86 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:38,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:38,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/13ff14b732e74380a329a2d11d5fe685 2024-04-14T11:16:38,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/3edc416d1bb5421aab36b95328a1d059 is 50, key is test_row_0/C:col10/1713093397022/Put/seqid=0 2024-04-14T11:16:38,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/3edc416d1bb5421aab36b95328a1d059 2024-04-14T11:16:38,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/71e1e4ed9c1a4de797a22b50e1e2f4a6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/71e1e4ed9c1a4de797a22b50e1e2f4a6 2024-04-14T11:16:38,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/71e1e4ed9c1a4de797a22b50e1e2f4a6, entries=150, sequenceid=491, filesize=12.0 K 2024-04-14T11:16:38,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/13ff14b732e74380a329a2d11d5fe685 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/13ff14b732e74380a329a2d11d5fe685 2024-04-14T11:16:38,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/13ff14b732e74380a329a2d11d5fe685, entries=150, sequenceid=491, filesize=12.0 K 2024-04-14T11:16:38,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/3edc416d1bb5421aab36b95328a1d059 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/3edc416d1bb5421aab36b95328a1d059 2024-04-14T11:16:38,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/3edc416d1bb5421aab36b95328a1d059, entries=150, sequenceid=491, filesize=12.0 K 2024-04-14T11:16:38,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for c0901aa47578fba1a7f58734a4a0e3b2 in 876ms, sequenceid=491, compaction requested=true 2024-04-14T11:16:38,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:38,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:38,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:38,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:38,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:38,509 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:38,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:38,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:38,509 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:38,511 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37908 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:38,511 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/A is initiating minor compaction (all files) 2024-04-14T11:16:38,511 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/A in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:38,511 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37908 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:38,511 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/9e8292ff3838453faaad98f977320699, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/e18f709ce3c940c7b3bb4cc9e59e3ffa, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/71e1e4ed9c1a4de797a22b50e1e2f4a6] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=37.0 K 2024-04-14T11:16:38,511 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/B is initiating minor compaction (all files) 2024-04-14T11:16:38,511 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/B in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:38,511 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/e37b04c2cbdd40c4ad5555a9f649149a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/402b9cb8178b46e7935beef58047f2f2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/13ff14b732e74380a329a2d11d5fe685] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=37.0 K 2024-04-14T11:16:38,512 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9e8292ff3838453faaad98f977320699, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1713093395260 2024-04-14T11:16:38,512 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting e37b04c2cbdd40c4ad5555a9f649149a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1713093395260 2024-04-14T11:16:38,512 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting e18f709ce3c940c7b3bb4cc9e59e3ffa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1713093395876 2024-04-14T11:16:38,512 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 402b9cb8178b46e7935beef58047f2f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1713093395876 2024-04-14T11:16:38,513 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 71e1e4ed9c1a4de797a22b50e1e2f4a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1713093397022 2024-04-14T11:16:38,513 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 13ff14b732e74380a329a2d11d5fe685, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1713093397022 2024-04-14T11:16:38,520 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#A#compaction#316 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:38,520 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#B#compaction#317 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:38,521 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/d595ad72770248a295c47b79fd9167c2 is 50, key is test_row_0/A:col10/1713093397022/Put/seqid=0 2024-04-14T11:16:38,521 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/a5c7a1207e304fefa3f4d6f232f5c14b is 50, key is test_row_0/B:col10/1713093397022/Put/seqid=0 2024-04-14T11:16:38,536 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/d595ad72770248a295c47b79fd9167c2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/d595ad72770248a295c47b79fd9167c2 2024-04-14T11:16:38,538 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/a5c7a1207e304fefa3f4d6f232f5c14b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/a5c7a1207e304fefa3f4d6f232f5c14b 2024-04-14T11:16:38,544 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/A of c0901aa47578fba1a7f58734a4a0e3b2 into d595ad72770248a295c47b79fd9167c2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:38,544 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:38,544 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/A, priority=13, startTime=1713093398509; duration=0sec 2024-04-14T11:16:38,545 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:38,545 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:A 2024-04-14T11:16:38,545 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:38,546 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/B of c0901aa47578fba1a7f58734a4a0e3b2 into a5c7a1207e304fefa3f4d6f232f5c14b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:38,546 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:38,546 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/B, priority=13, startTime=1713093398509; duration=0sec 2024-04-14T11:16:38,546 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:38,546 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:B 2024-04-14T11:16:38,546 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37908 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:38,546 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/C is initiating minor compaction (all files) 2024-04-14T11:16:38,546 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/C in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:38,547 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/dcbe2fe3dcc94cbfb36405844f1f6bea, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/736b854f227f40159ff1c8876e7a3e37, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/3edc416d1bb5421aab36b95328a1d059] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=37.0 K 2024-04-14T11:16:38,547 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting dcbe2fe3dcc94cbfb36405844f1f6bea, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1713093395260 2024-04-14T11:16:38,548 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 736b854f227f40159ff1c8876e7a3e37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1713093395876 2024-04-14T11:16:38,548 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3edc416d1bb5421aab36b95328a1d059, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1713093397022 2024-04-14T11:16:38,558 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#C#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:38,558 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/991fc56df8254f4a9a4cab76fc3cdc8d is 50, key is test_row_0/C:col10/1713093397022/Put/seqid=0 2024-04-14T11:16:38,584 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/991fc56df8254f4a9a4cab76fc3cdc8d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/991fc56df8254f4a9a4cab76fc3cdc8d 2024-04-14T11:16:38,590 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/C of c0901aa47578fba1a7f58734a4a0e3b2 into 991fc56df8254f4a9a4cab76fc3cdc8d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:38,590 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:38,590 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/C, priority=13, startTime=1713093398509; duration=0sec 2024-04-14T11:16:38,590 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:38,590 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:C 2024-04-14T11:16:38,601 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:38,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-04-14T11:16:38,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:38,602 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-04-14T11:16:38,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/a7a05612b1224d96988a95f0a1c41ee2 is 50, key is test_row_0/A:col10/1713093397640/Put/seqid=0 2024-04-14T11:16:38,612 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/a7a05612b1224d96988a95f0a1c41ee2 2024-04-14T11:16:38,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/b021bd1ba6b249e38e1708cc19bf87f6 is 50, key is test_row_0/B:col10/1713093397640/Put/seqid=0 2024-04-14T11:16:38,625 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/b021bd1ba6b249e38e1708cc19bf87f6 2024-04-14T11:16:38,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/c934183050c646d5a3ed71e494392a19 is 50, key is test_row_0/C:col10/1713093397640/Put/seqid=0 2024-04-14T11:16:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:38,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:38,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:38,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093458769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:38,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093458771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:38,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:38,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093458771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:38,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093458771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093458772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:38,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=85 2024-04-14T11:16:38,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:38,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093458873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:38,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093458875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:38,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093458875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:38,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:38,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093458875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093458875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,046 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/c934183050c646d5a3ed71e494392a19 2024-04-14T11:16:39,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/a7a05612b1224d96988a95f0a1c41ee2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/a7a05612b1224d96988a95f0a1c41ee2 2024-04-14T11:16:39,060 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/a7a05612b1224d96988a95f0a1c41ee2, entries=150, sequenceid=509, filesize=12.0 K 2024-04-14T11:16:39,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/b021bd1ba6b249e38e1708cc19bf87f6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/b021bd1ba6b249e38e1708cc19bf87f6 2024-04-14T11:16:39,069 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/b021bd1ba6b249e38e1708cc19bf87f6, entries=150, sequenceid=509, filesize=12.0 K 2024-04-14T11:16:39,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/c934183050c646d5a3ed71e494392a19 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c934183050c646d5a3ed71e494392a19 2024-04-14T11:16:39,078 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c934183050c646d5a3ed71e494392a19, entries=150, sequenceid=509, filesize=12.0 K 2024-04-14T11:16:39,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093459077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,080 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(3035): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for c0901aa47578fba1a7f58734a4a0e3b2 in 478ms, sequenceid=509, compaction requested=false 2024-04-14T11:16:39,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:39,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:39,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=86 2024-04-14T11:16:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:39,080 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:39,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-04-14T11:16:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=86 2024-04-14T11:16:39,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=86, resume processing ppid=85 2024-04-14T11:16:39,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=86, ppid=85, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3980 sec 2024-04-14T11:16:39,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=85, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees in 1.4070 sec 2024-04-14T11:16:39,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/2dffebac1297446b86732c6d5f5e1105 is 50, key is test_row_0/A:col10/1713093398770/Put/seqid=0 2024-04-14T11:16:39,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093459099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093459104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093459106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093459106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093459208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093459214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093459214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093459214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093459382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093459414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093459420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093459420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093459422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/2dffebac1297446b86732c6d5f5e1105 2024-04-14T11:16:39,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/0122aa6d9b604519ab1cfbcbae191aaa is 50, key is test_row_0/B:col10/1713093398770/Put/seqid=0 2024-04-14T11:16:39,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/0122aa6d9b604519ab1cfbcbae191aaa 2024-04-14T11:16:39,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/d757003730e04cb5bc43a7d4e0508e66 is 50, key is test_row_0/C:col10/1713093398770/Put/seqid=0 2024-04-14T11:16:39,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/d757003730e04cb5bc43a7d4e0508e66 2024-04-14T11:16:39,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/2dffebac1297446b86732c6d5f5e1105 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2dffebac1297446b86732c6d5f5e1105 2024-04-14T11:16:39,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2dffebac1297446b86732c6d5f5e1105, entries=200, sequenceid=531, filesize=14.4 K 2024-04-14T11:16:39,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/0122aa6d9b604519ab1cfbcbae191aaa as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/0122aa6d9b604519ab1cfbcbae191aaa 2024-04-14T11:16:39,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/0122aa6d9b604519ab1cfbcbae191aaa, entries=150, sequenceid=531, filesize=12.0 K 2024-04-14T11:16:39,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/d757003730e04cb5bc43a7d4e0508e66 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/d757003730e04cb5bc43a7d4e0508e66 2024-04-14T11:16:39,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/d757003730e04cb5bc43a7d4e0508e66, entries=150, sequenceid=531, filesize=12.0 K 2024-04-14T11:16:39,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for c0901aa47578fba1a7f58734a4a0e3b2 in 521ms, sequenceid=531, compaction requested=true 2024-04-14T11:16:39,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:39,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:39,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:39,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:39,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:39,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:39,601 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:39,601 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:39,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:39,603 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38010 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:39,603 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/B is initiating minor compaction (all files) 2024-04-14T11:16:39,603 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/B in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:39,603 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/a5c7a1207e304fefa3f4d6f232f5c14b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/b021bd1ba6b249e38e1708cc19bf87f6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/0122aa6d9b604519ab1cfbcbae191aaa] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=37.1 K 2024-04-14T11:16:39,603 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40450 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:39,603 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/A is initiating minor compaction (all files) 2024-04-14T11:16:39,603 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/A in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:39,603 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/d595ad72770248a295c47b79fd9167c2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/a7a05612b1224d96988a95f0a1c41ee2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2dffebac1297446b86732c6d5f5e1105] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=39.5 K 2024-04-14T11:16:39,603 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting a5c7a1207e304fefa3f4d6f232f5c14b, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1713093397022 2024-04-14T11:16:39,604 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting d595ad72770248a295c47b79fd9167c2, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1713093397022 2024-04-14T11:16:39,604 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting b021bd1ba6b249e38e1708cc19bf87f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1713093397640 2024-04-14T11:16:39,604 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting a7a05612b1224d96988a95f0a1c41ee2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1713093397640 2024-04-14T11:16:39,605 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2dffebac1297446b86732c6d5f5e1105, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1713093398764 2024-04-14T11:16:39,605 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 0122aa6d9b604519ab1cfbcbae191aaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1713093398770 2024-04-14T11:16:39,618 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#B#compaction#325 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:39,618 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/50517c0199da46639ee84a9307962fa8 is 50, key is test_row_0/B:col10/1713093398770/Put/seqid=0 2024-04-14T11:16:39,621 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#A#compaction#326 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:39,621 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/ed07234e435d4146a2239ef7e0d19d9d is 50, key is test_row_0/A:col10/1713093398770/Put/seqid=0 2024-04-14T11:16:39,641 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/ed07234e435d4146a2239ef7e0d19d9d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/ed07234e435d4146a2239ef7e0d19d9d 2024-04-14T11:16:39,649 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/A of c0901aa47578fba1a7f58734a4a0e3b2 into ed07234e435d4146a2239ef7e0d19d9d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:39,649 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:39,649 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/A, priority=13, startTime=1713093399601; duration=0sec 2024-04-14T11:16:39,649 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:39,649 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:A 2024-04-14T11:16:39,649 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:39,655 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38010 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:39,655 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/C is initiating minor compaction (all files) 2024-04-14T11:16:39,655 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/C in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:39,655 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/991fc56df8254f4a9a4cab76fc3cdc8d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c934183050c646d5a3ed71e494392a19, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/d757003730e04cb5bc43a7d4e0508e66] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=37.1 K 2024-04-14T11:16:39,658 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 991fc56df8254f4a9a4cab76fc3cdc8d, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1713093397022 2024-04-14T11:16:39,659 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c934183050c646d5a3ed71e494392a19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1713093397640 2024-04-14T11:16:39,661 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting d757003730e04cb5bc43a7d4e0508e66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1713093398770 2024-04-14T11:16:39,682 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#C#compaction#327 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:39,683 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/7349d7d2ff744625b810f5524172f512 is 50, key is test_row_0/C:col10/1713093398770/Put/seqid=0 2024-04-14T11:16:39,722 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:39,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-04-14T11:16:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:39,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/37aa913372514c89b6f0450243c55656 is 50, key is test_row_0/A:col10/1713093399102/Put/seqid=0 2024-04-14T11:16:39,737 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/7349d7d2ff744625b810f5524172f512 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/7349d7d2ff744625b810f5524172f512 2024-04-14T11:16:39,747 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/C of c0901aa47578fba1a7f58734a4a0e3b2 into 7349d7d2ff744625b810f5524172f512(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:39,747 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:39,747 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/C, priority=13, startTime=1713093399601; duration=0sec 2024-04-14T11:16:39,747 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:39,747 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:C 2024-04-14T11:16:39,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093459749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093459752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093459752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093459756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=85 2024-04-14T11:16:39,788 INFO [Thread-1198 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 85 completed 2024-04-14T11:16:39,791 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-04-14T11:16:39,796 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=87 2024-04-14T11:16:39,797 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:39,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:39,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093459857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093459858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093459858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093459860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58188 deadline: 1713093459886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=87 2024-04-14T11:16:39,954 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:39,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-04-14T11:16:39,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:39,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:39,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:39,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(53): pid=88 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:39,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:40,042 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/50517c0199da46639ee84a9307962fa8 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/50517c0199da46639ee84a9307962fa8 2024-04-14T11:16:40,049 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/B of c0901aa47578fba1a7f58734a4a0e3b2 into 50517c0199da46639ee84a9307962fa8(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:40,049 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:40,049 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/B, priority=13, startTime=1713093399601; duration=0sec 2024-04-14T11:16:40,049 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:40,049 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:B 2024-04-14T11:16:40,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093460062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:40,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:40,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093460061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093460062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:40,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093460064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=87 2024-04-14T11:16:40,108 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:40,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-04-14T11:16:40,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:40,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:40,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:40,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(53): pid=88 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:40,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:40,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/37aa913372514c89b6f0450243c55656 2024-04-14T11:16:40,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/71f8a19969f34b3e8a6ed864ca31bb9f is 50, key is test_row_0/B:col10/1713093399102/Put/seqid=0 2024-04-14T11:16:40,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/71f8a19969f34b3e8a6ed864ca31bb9f 2024-04-14T11:16:40,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/f2ad9e36e6f54119bf7572b270878f40 is 50, key is test_row_0/C:col10/1713093399102/Put/seqid=0 2024-04-14T11:16:40,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/f2ad9e36e6f54119bf7572b270878f40 2024-04-14T11:16:40,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/37aa913372514c89b6f0450243c55656 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/37aa913372514c89b6f0450243c55656 2024-04-14T11:16:40,262 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:40,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-04-14T11:16:40,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:40,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:40,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:40,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(53): pid=88 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:40,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:40,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/37aa913372514c89b6f0450243c55656, entries=150, sequenceid=548, filesize=12.0 K 2024-04-14T11:16:40,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/71f8a19969f34b3e8a6ed864ca31bb9f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/71f8a19969f34b3e8a6ed864ca31bb9f 2024-04-14T11:16:40,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/71f8a19969f34b3e8a6ed864ca31bb9f, entries=150, sequenceid=548, filesize=12.0 K 2024-04-14T11:16:40,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/f2ad9e36e6f54119bf7572b270878f40 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/f2ad9e36e6f54119bf7572b270878f40 2024-04-14T11:16:40,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/f2ad9e36e6f54119bf7572b270878f40, entries=150, sequenceid=548, filesize=12.0 K 2024-04-14T11:16:40,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for c0901aa47578fba1a7f58734a4a0e3b2 in 574ms, sequenceid=548, compaction requested=false 2024-04-14T11:16:40,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:40,372 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:40,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-04-14T11:16:40,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/3258573f955b45bc84b08182249956e1 is 50, key is test_row_0/A:col10/1713093399754/Put/seqid=0 2024-04-14T11:16:40,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/3258573f955b45bc84b08182249956e1 2024-04-14T11:16:40,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093460391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:40,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093460394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:40,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093460395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:40,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093460395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=87 2024-04-14T11:16:40,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/253d3679bc45456ba2348e080fe576ad is 50, key is test_row_0/B:col10/1713093399754/Put/seqid=0 2024-04-14T11:16:40,416 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:40,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-04-14T11:16:40,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:40,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:40,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:40,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(53): pid=88 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:40,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:40,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:40,450 WARN [DataStreamer for file /user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/253d3679bc45456ba2348e080fe576ad {}] hdfs.DataStreamer(982): Caught exception java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:1.8.0_362] at java.lang.Thread.join(Thread.java:1257) ~[?:1.8.0_362] at java.lang.Thread.join(Thread.java:1331) ~[?:1.8.0_362] at org.apache.hadoop.hdfs.DataStreamer.closeResponder(DataStreamer.java:980) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DataStreamer.endBlock(DataStreamer.java:630) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:807) ~[hadoop-hdfs-client-2.10.2.jar:?] 2024-04-14T11:16:40,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/253d3679bc45456ba2348e080fe576ad 2024-04-14T11:16:40,455 DEBUG [Thread-1201 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x76a586b1 to 127.0.0.1:60913 2024-04-14T11:16:40,456 DEBUG [Thread-1201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:40,456 DEBUG [Thread-1199 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x41bcfd04 to 127.0.0.1:60913 2024-04-14T11:16:40,456 DEBUG [Thread-1199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:40,456 DEBUG [Thread-1205 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x374b3f29 to 127.0.0.1:60913 2024-04-14T11:16:40,456 DEBUG [Thread-1205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:40,458 DEBUG [Thread-1203 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x349e56ad to 127.0.0.1:60913 2024-04-14T11:16:40,458 DEBUG [Thread-1203 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:40,459 DEBUG [Thread-1207 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0cdd2dc7 to 127.0.0.1:60913 2024-04-14T11:16:40,459 DEBUG [Thread-1207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:40,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/43f54c2918a4499f914b24b8b3fea2d5 is 50, key is test_row_0/C:col10/1713093399754/Put/seqid=0 2024-04-14T11:16:40,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/43f54c2918a4499f914b24b8b3fea2d5 2024-04-14T11:16:40,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/3258573f955b45bc84b08182249956e1 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/3258573f955b45bc84b08182249956e1 2024-04-14T11:16:40,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/3258573f955b45bc84b08182249956e1, entries=200, sequenceid=572, filesize=14.4 K 2024-04-14T11:16:40,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58166 deadline: 1713093460500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:40,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58136 deadline: 1713093460500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:40,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58132 deadline: 1713093460500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:40,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/253d3679bc45456ba2348e080fe576ad as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/253d3679bc45456ba2348e080fe576ad 2024-04-14T11:16:40,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:40,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:58122 deadline: 1713093460502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:40,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/253d3679bc45456ba2348e080fe576ad, entries=150, sequenceid=572, filesize=12.0 K 2024-04-14T11:16:40,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/43f54c2918a4499f914b24b8b3fea2d5 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/43f54c2918a4499f914b24b8b3fea2d5 2024-04-14T11:16:40,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/43f54c2918a4499f914b24b8b3fea2d5, entries=150, sequenceid=572, filesize=12.0 K 2024-04-14T11:16:40,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for c0901aa47578fba1a7f58734a4a0e3b2 in 146ms, sequenceid=572, compaction requested=true 2024-04-14T11:16:40,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:40,518 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:40,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:40,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:40,519 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:40,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:40,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:40,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0901aa47578fba1a7f58734a4a0e3b2:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:40,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:40,520 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40552 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:40,520 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/A is initiating minor compaction (all files) 2024-04-14T11:16:40,520 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/A in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:40,520 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/ed07234e435d4146a2239ef7e0d19d9d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/37aa913372514c89b6f0450243c55656, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/3258573f955b45bc84b08182249956e1] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=39.6 K 2024-04-14T11:16:40,521 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting ed07234e435d4146a2239ef7e0d19d9d, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1713093398770 2024-04-14T11:16:40,521 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38112 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:40,521 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/B is initiating minor compaction (all files) 2024-04-14T11:16:40,521 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/B in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:40,521 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/50517c0199da46639ee84a9307962fa8, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/71f8a19969f34b3e8a6ed864ca31bb9f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/253d3679bc45456ba2348e080fe576ad] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=37.2 K 2024-04-14T11:16:40,521 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 37aa913372514c89b6f0450243c55656, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1713093399099 2024-04-14T11:16:40,522 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 50517c0199da46639ee84a9307962fa8, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1713093398770 2024-04-14T11:16:40,522 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3258573f955b45bc84b08182249956e1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1713093399747 2024-04-14T11:16:40,522 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 71f8a19969f34b3e8a6ed864ca31bb9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1713093399099 2024-04-14T11:16:40,523 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 253d3679bc45456ba2348e080fe576ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1713093399754 2024-04-14T11:16:40,532 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#A#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:40,532 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#B#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:40,532 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/108d47ae36534e2abc8716541a260cf2 is 50, key is test_row_0/B:col10/1713093399754/Put/seqid=0 2024-04-14T11:16:40,533 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/5daeb26b09e54714bcc5ddaa30647e86 is 50, key is test_row_0/A:col10/1713093399754/Put/seqid=0 2024-04-14T11:16:40,554 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/5daeb26b09e54714bcc5ddaa30647e86 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5daeb26b09e54714bcc5ddaa30647e86 2024-04-14T11:16:40,571 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:40,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-04-14T11:16:40,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:40,578 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-04-14T11:16:40,586 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/A of c0901aa47578fba1a7f58734a4a0e3b2 into 5daeb26b09e54714bcc5ddaa30647e86(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:40,586 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:40,586 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/A, priority=13, startTime=1713093400518; duration=0sec 2024-04-14T11:16:40,587 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:40,587 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:A 2024-04-14T11:16:40,587 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:40,588 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38112 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:40,588 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): c0901aa47578fba1a7f58734a4a0e3b2/C is initiating minor compaction (all files) 2024-04-14T11:16:40,588 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of c0901aa47578fba1a7f58734a4a0e3b2/C in TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:40,589 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/7349d7d2ff744625b810f5524172f512, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/f2ad9e36e6f54119bf7572b270878f40, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/43f54c2918a4499f914b24b8b3fea2d5] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp, totalSize=37.2 K 2024-04-14T11:16:40,589 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7349d7d2ff744625b810f5524172f512, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1713093398770 2024-04-14T11:16:40,590 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting f2ad9e36e6f54119bf7572b270878f40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1713093399099 2024-04-14T11:16:40,591 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 43f54c2918a4499f914b24b8b3fea2d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=572, earliestPutTs=1713093399754 2024-04-14T11:16:40,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/04cb06f0ae48452cbd10deda0f5435bf is 50, key is test_row_0/A:col10/1713093400393/Put/seqid=0 2024-04-14T11:16:40,611 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0901aa47578fba1a7f58734a4a0e3b2#C#compaction#337 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:40,612 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/2721f4980e1d4175a7cf8182f33381f7 is 50, key is test_row_0/C:col10/1713093399754/Put/seqid=0 2024-04-14T11:16:40,639 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/04cb06f0ae48452cbd10deda0f5435bf 2024-04-14T11:16:40,643 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/2721f4980e1d4175a7cf8182f33381f7 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2721f4980e1d4175a7cf8182f33381f7 2024-04-14T11:16:40,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/4ef0d101026746ba9a9ad19776d23656 is 50, key is test_row_0/B:col10/1713093400393/Put/seqid=0 2024-04-14T11:16:40,656 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/C of c0901aa47578fba1a7f58734a4a0e3b2 into 2721f4980e1d4175a7cf8182f33381f7(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:40,656 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:40,656 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/C, priority=13, startTime=1713093400519; duration=0sec 2024-04-14T11:16:40,656 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:40,656 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:C 2024-04-14T11:16:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:40,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. as already flushing 2024-04-14T11:16:40,707 DEBUG [Thread-1188 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5e76c41d to 127.0.0.1:60913 2024-04-14T11:16:40,707 DEBUG [Thread-1188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:40,708 DEBUG [Thread-1192 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4dd5fe20 to 127.0.0.1:60913 2024-04-14T11:16:40,708 DEBUG [Thread-1192 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:40,710 DEBUG [Thread-1194 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3d752bcd to 127.0.0.1:60913 2024-04-14T11:16:40,710 DEBUG [Thread-1194 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:40,711 DEBUG [Thread-1190 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0dd7f4f1 to 127.0.0.1:60913 2024-04-14T11:16:40,711 DEBUG [Thread-1190 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:40,898 DEBUG [Thread-1196 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2ceda901 to 127.0.0.1:60913 2024-04-14T11:16:40,898 DEBUG [Thread-1196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:40,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=87 2024-04-14T11:16:40,956 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/108d47ae36534e2abc8716541a260cf2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/108d47ae36534e2abc8716541a260cf2 2024-04-14T11:16:40,965 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in c0901aa47578fba1a7f58734a4a0e3b2/B of c0901aa47578fba1a7f58734a4a0e3b2 into 108d47ae36534e2abc8716541a260cf2(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:40,965 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:40,965 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2., storeName=c0901aa47578fba1a7f58734a4a0e3b2/B, priority=13, startTime=1713093400519; duration=0sec 2024-04-14T11:16:40,965 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:40,965 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0901aa47578fba1a7f58734a4a0e3b2:B 2024-04-14T11:16:41,075 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/4ef0d101026746ba9a9ad19776d23656 2024-04-14T11:16:41,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/0872c70f417a415cb123aa83ceed654a is 50, key is test_row_0/C:col10/1713093400393/Put/seqid=0 2024-04-14T11:16:41,098 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/0872c70f417a415cb123aa83ceed654a 2024-04-14T11:16:41,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/04cb06f0ae48452cbd10deda0f5435bf as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/04cb06f0ae48452cbd10deda0f5435bf 2024-04-14T11:16:41,123 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/04cb06f0ae48452cbd10deda0f5435bf, entries=100, sequenceid=588, filesize=9.6 K 2024-04-14T11:16:41,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/4ef0d101026746ba9a9ad19776d23656 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/4ef0d101026746ba9a9ad19776d23656 2024-04-14T11:16:41,137 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/4ef0d101026746ba9a9ad19776d23656, entries=100, sequenceid=588, filesize=9.6 K 2024-04-14T11:16:41,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/0872c70f417a415cb123aa83ceed654a as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/0872c70f417a415cb123aa83ceed654a 2024-04-14T11:16:41,147 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/0872c70f417a415cb123aa83ceed654a, entries=100, sequenceid=588, filesize=9.6 K 2024-04-14T11:16:41,151 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3035): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=33.54 KB/34350 for c0901aa47578fba1a7f58734a4a0e3b2 in 573ms, sequenceid=588, compaction requested=false 2024-04-14T11:16:41,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2533): Flush status journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:41,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:41,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=88 2024-04-14T11:16:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=88 2024-04-14T11:16:41,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=88, resume processing ppid=87 2024-04-14T11:16:41,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=88, ppid=87, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3570 sec 2024-04-14T11:16:41,158 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=87, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 1.3660 sec 2024-04-14T11:16:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=87 2024-04-14T11:16:41,904 INFO [Thread-1198 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 87 completed 2024-04-14T11:16:41,904 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-04-14T11:16:41,904 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-04-14T11:16:41,904 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-04-14T11:16:41,904 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-04-14T11:16:41,904 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-04-14T11:16:41,904 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-04-14T11:16:41,904 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-04-14T11:16:41,904 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(398): read 5032 2024-04-14T11:16:41,904 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(398): read 4979 2024-04-14T11:16:41,904 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(398): read 4928 2024-04-14T11:16:41,905 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(398): read 5005 2024-04-14T11:16:41,905 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(398): read 4983 2024-04-14T11:16:41,905 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-04-14T11:16:41,905 INFO [Listener at localhost/34025 {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-04-14T11:16:41,905 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2e20f4a4 to 127.0.0.1:60913 2024-04-14T11:16:41,905 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:16:41,906 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$18(968): Started disable of TestAcidGuarantees 2024-04-14T11:16:41,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$13(2744): Client=jenkins//172.31.2.20 disable TestAcidGuarantees 2024-04-14T11:16:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=89, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-04-14T11:16:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2024-04-14T11:16:41,911 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093401911"}]},"ts":"1713093401911"} 2024-04-14T11:16:41,913 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-04-14T11:16:41,916 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set TestAcidGuarantees to state=DISABLING 2024-04-14T11:16:41,916 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0901aa47578fba1a7f58734a4a0e3b2, UNASSIGN}] 2024-04-14T11:16:41,917 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(729): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0901aa47578fba1a7f58734a4a0e3b2, UNASSIGN 2024-04-14T11:16:41,918 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=90 updating hbase:meta row=c0901aa47578fba1a7f58734a4a0e3b2, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:41,919 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE; CloseRegionProcedure c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:16:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2024-04-14T11:16:42,071 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:42,072 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(119): Close c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:42,072 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Closing c0901aa47578fba1a7f58734a4a0e3b2, disabling compactions & flushes 2024-04-14T11:16:42,072 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1698): Closing region TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:42,072 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1719): Waiting without time limit for close lock on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:42,072 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1786): Acquired close lock on TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. after waiting 0 ms 2024-04-14T11:16:42,072 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1796): Updates disabled for region TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:42,072 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(2832): Flushing c0901aa47578fba1a7f58734a4a0e3b2 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-04-14T11:16:42,076 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/51592514965a49019948bdfdb3abf649 is 50, key is test_row_0/A:col10/1713093400704/Put/seqid=0 2024-04-14T11:16:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2024-04-14T11:16:42,481 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/51592514965a49019948bdfdb3abf649 2024-04-14T11:16:42,488 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/6ef693781572480ead07b75ac8e14db9 is 50, key is test_row_0/B:col10/1713093400704/Put/seqid=0 2024-04-14T11:16:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2024-04-14T11:16:42,893 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/6ef693781572480ead07b75ac8e14db9 2024-04-14T11:16:42,901 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/bcb99aec2e13436886287472848298c2 is 50, key is test_row_0/C:col10/1713093400704/Put/seqid=0 2024-04-14T11:16:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2024-04-14T11:16:43,307 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/bcb99aec2e13436886287472848298c2 2024-04-14T11:16:43,313 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/A/51592514965a49019948bdfdb3abf649 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/51592514965a49019948bdfdb3abf649 2024-04-14T11:16:43,318 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/51592514965a49019948bdfdb3abf649, entries=150, sequenceid=598, filesize=12.0 K 2024-04-14T11:16:43,319 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/B/6ef693781572480ead07b75ac8e14db9 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/6ef693781572480ead07b75ac8e14db9 2024-04-14T11:16:43,323 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/6ef693781572480ead07b75ac8e14db9, entries=150, sequenceid=598, filesize=12.0 K 2024-04-14T11:16:43,324 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/.tmp/C/bcb99aec2e13436886287472848298c2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/bcb99aec2e13436886287472848298c2 2024-04-14T11:16:43,327 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/bcb99aec2e13436886287472848298c2, entries=150, sequenceid=598, filesize=12.0 K 2024-04-14T11:16:43,328 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(3035): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for c0901aa47578fba1a7f58734a4a0e3b2 in 1256ms, sequenceid=598, compaction requested=true 2024-04-14T11:16:43,342 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/b5f0e269708f4170b0266c2ee661c20c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2dc3830742044f47b3f5ab267b3a1a41, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/815ab35dc91f4bf4abbe57600fe1c45d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2b71dbcc28df4fd9a425065b9bdee307, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/37b3eb2b2dc44c5fa5f7db11f06111dc, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/d25b9b0cb1114155af3bf581c0ccc75b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5f8f85adda584a2e8e4e7d90ae2effe4, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/cfc84c13ae7a46e6b8a4f2d9c8eb1630, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5ad935177c7f4904b4d37f272e939792, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/c54ef7815aa942318e05332e54204a8d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/709804e770ec450c8b996438f12e07d3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/332f364af45c42c2a1f20db1f7d9405b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5bc39d8ddc484a6cb9973b21cd311467, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5527ee4b31e143438793ef0c9931e0bb, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/52c5cfd07d134cf8bce3913aa1fc75db, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/224e720eaea245e4ba45547f7d495cda, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/c9264ba28c884b689b4f63fe2935dd1b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/df1526798ebd457b9b43f3a36f833639, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/dab56de4ef2c4dbfa7f3d69119c56e32, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/8d692a9e7ce74e659330bf7dfa416df6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/b3762db97a03466ba358562a058873ce, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/cf95a32f5a9c458bb70610814c5d4ec9, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/eed8dd27410046158e6f8c7236436d28, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/bad60be8bfce4955aa80d3a2184b04fd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/fcb4249c23454cd4a652cba4dcddaa30, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2a52e0300bb84abe998e8013e3496141, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/7dbc5dd0a638447db170696dcc3f9931, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/099e610cccb8461486e1a927665cd335, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/a913205557ff425b8eb03117d768454e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5c0cbbaa750f46318170d40bde0e4c5c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/9e8292ff3838453faaad98f977320699, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5085c5eaab55467e9cd93d9875b6f979, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/e18f709ce3c940c7b3bb4cc9e59e3ffa, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/d595ad72770248a295c47b79fd9167c2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/71e1e4ed9c1a4de797a22b50e1e2f4a6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/a7a05612b1224d96988a95f0a1c41ee2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2dffebac1297446b86732c6d5f5e1105, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/ed07234e435d4146a2239ef7e0d19d9d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/37aa913372514c89b6f0450243c55656, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/3258573f955b45bc84b08182249956e1] to archive 2024-04-14T11:16:43,344 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:16:43,346 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/b5f0e269708f4170b0266c2ee661c20c to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/b5f0e269708f4170b0266c2ee661c20c 2024-04-14T11:16:43,347 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2dc3830742044f47b3f5ab267b3a1a41 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2dc3830742044f47b3f5ab267b3a1a41 2024-04-14T11:16:43,349 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/815ab35dc91f4bf4abbe57600fe1c45d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/815ab35dc91f4bf4abbe57600fe1c45d 2024-04-14T11:16:43,350 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2b71dbcc28df4fd9a425065b9bdee307 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2b71dbcc28df4fd9a425065b9bdee307 2024-04-14T11:16:43,351 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/37b3eb2b2dc44c5fa5f7db11f06111dc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/37b3eb2b2dc44c5fa5f7db11f06111dc 2024-04-14T11:16:43,353 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/d25b9b0cb1114155af3bf581c0ccc75b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/d25b9b0cb1114155af3bf581c0ccc75b 2024-04-14T11:16:43,354 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5f8f85adda584a2e8e4e7d90ae2effe4 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5f8f85adda584a2e8e4e7d90ae2effe4 2024-04-14T11:16:43,355 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/cfc84c13ae7a46e6b8a4f2d9c8eb1630 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/cfc84c13ae7a46e6b8a4f2d9c8eb1630 2024-04-14T11:16:43,356 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5ad935177c7f4904b4d37f272e939792 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5ad935177c7f4904b4d37f272e939792 2024-04-14T11:16:43,357 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/c54ef7815aa942318e05332e54204a8d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/c54ef7815aa942318e05332e54204a8d 2024-04-14T11:16:43,359 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/709804e770ec450c8b996438f12e07d3 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/709804e770ec450c8b996438f12e07d3 2024-04-14T11:16:43,360 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/332f364af45c42c2a1f20db1f7d9405b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/332f364af45c42c2a1f20db1f7d9405b 2024-04-14T11:16:43,361 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5bc39d8ddc484a6cb9973b21cd311467 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5bc39d8ddc484a6cb9973b21cd311467 2024-04-14T11:16:43,362 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5527ee4b31e143438793ef0c9931e0bb to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5527ee4b31e143438793ef0c9931e0bb 2024-04-14T11:16:43,363 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/52c5cfd07d134cf8bce3913aa1fc75db to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/52c5cfd07d134cf8bce3913aa1fc75db 2024-04-14T11:16:43,364 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/224e720eaea245e4ba45547f7d495cda to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/224e720eaea245e4ba45547f7d495cda 2024-04-14T11:16:43,366 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/c9264ba28c884b689b4f63fe2935dd1b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/c9264ba28c884b689b4f63fe2935dd1b 2024-04-14T11:16:43,367 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/df1526798ebd457b9b43f3a36f833639 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/df1526798ebd457b9b43f3a36f833639 2024-04-14T11:16:43,368 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/dab56de4ef2c4dbfa7f3d69119c56e32 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/dab56de4ef2c4dbfa7f3d69119c56e32 2024-04-14T11:16:43,370 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/8d692a9e7ce74e659330bf7dfa416df6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/8d692a9e7ce74e659330bf7dfa416df6 2024-04-14T11:16:43,371 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/b3762db97a03466ba358562a058873ce to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/b3762db97a03466ba358562a058873ce 2024-04-14T11:16:43,372 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/cf95a32f5a9c458bb70610814c5d4ec9 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/cf95a32f5a9c458bb70610814c5d4ec9 2024-04-14T11:16:43,373 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/eed8dd27410046158e6f8c7236436d28 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/eed8dd27410046158e6f8c7236436d28 2024-04-14T11:16:43,375 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/bad60be8bfce4955aa80d3a2184b04fd to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/bad60be8bfce4955aa80d3a2184b04fd 2024-04-14T11:16:43,376 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/fcb4249c23454cd4a652cba4dcddaa30 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/fcb4249c23454cd4a652cba4dcddaa30 2024-04-14T11:16:43,377 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2a52e0300bb84abe998e8013e3496141 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2a52e0300bb84abe998e8013e3496141 2024-04-14T11:16:43,378 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/7dbc5dd0a638447db170696dcc3f9931 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/7dbc5dd0a638447db170696dcc3f9931 2024-04-14T11:16:43,379 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/099e610cccb8461486e1a927665cd335 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/099e610cccb8461486e1a927665cd335 2024-04-14T11:16:43,381 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/a913205557ff425b8eb03117d768454e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/a913205557ff425b8eb03117d768454e 2024-04-14T11:16:43,382 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5c0cbbaa750f46318170d40bde0e4c5c to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5c0cbbaa750f46318170d40bde0e4c5c 2024-04-14T11:16:43,383 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/9e8292ff3838453faaad98f977320699 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/9e8292ff3838453faaad98f977320699 2024-04-14T11:16:43,384 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5085c5eaab55467e9cd93d9875b6f979 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5085c5eaab55467e9cd93d9875b6f979 2024-04-14T11:16:43,386 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/e18f709ce3c940c7b3bb4cc9e59e3ffa to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/e18f709ce3c940c7b3bb4cc9e59e3ffa 2024-04-14T11:16:43,387 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/d595ad72770248a295c47b79fd9167c2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/d595ad72770248a295c47b79fd9167c2 2024-04-14T11:16:43,388 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/71e1e4ed9c1a4de797a22b50e1e2f4a6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/71e1e4ed9c1a4de797a22b50e1e2f4a6 2024-04-14T11:16:43,389 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/a7a05612b1224d96988a95f0a1c41ee2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/a7a05612b1224d96988a95f0a1c41ee2 2024-04-14T11:16:43,391 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2dffebac1297446b86732c6d5f5e1105 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/2dffebac1297446b86732c6d5f5e1105 2024-04-14T11:16:43,392 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/ed07234e435d4146a2239ef7e0d19d9d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/ed07234e435d4146a2239ef7e0d19d9d 2024-04-14T11:16:43,393 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/37aa913372514c89b6f0450243c55656 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/37aa913372514c89b6f0450243c55656 2024-04-14T11:16:43,395 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/3258573f955b45bc84b08182249956e1 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/3258573f955b45bc84b08182249956e1 2024-04-14T11:16:43,396 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/abd04d31efdb4bb98d5dda74c65288b0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/788dc4c85ec4450fabcb862c7411afc4, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/3cc1f600dbb042b0ac3468a0eeefb702, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/9a84a28639f14d8c9f09dcd386c3226c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/04e29a382f9b4e57a1de33850a41cc1b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/d1e35a799d6b4cc486d97ca00857dba6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/95ab2d9391b044b681fb0aa3d12ba808, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/a3e8835ba40c4bb394e720fdd0df479c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/beb94d3fec3c44edb2e7fd854eeede31, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f6daee50e0eb453b9e280eb3416e25ed, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/8f8779931cdd404b8c88a8f1cb91eb50, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/5d5bd0f8a39b476d970be6b4dd0d5baa, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f855465c00404beeaae3d59dc8253de3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/3d7401081d6c452b9ba6c5ab204373f3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/a5e26fa4198141a9b3b0d132e45dbd42, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f87225c8929942abb75fd4010c2b2ad7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/be340246078e4e65a961087e1428fa5b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/840b2decca8946b1a065decae005847e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/296996bf35664c14888bde44ccf3d3ad, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/2c195e64efb14cff9ca4a7210ccb6821, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/d03d9de8bde64b5c9c591980dfaec1d5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/b4f8c2822ba24305b9629c2f211fb0e8, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/e26d1bdd030b451381ee04da62cc0490, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/1534a7d91d9c4352a012a263005b1fcc, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/cd1da2f729a044af91ad3f50f63ecd0b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/bd30a71ec07b4835be75d98ba18a8d8e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/8dd7b20164e94073b31037f685194213, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f73d48bd452f4fd0a7b3ce87de82a024, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/1e3c7aa35384457da9f4c1e8fb6c8760, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/c0366f52f3e245eda187348a72a77134, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/e37b04c2cbdd40c4ad5555a9f649149a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/ca387b1fb9dd44efba1ffae40531b3b7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/402b9cb8178b46e7935beef58047f2f2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/a5c7a1207e304fefa3f4d6f232f5c14b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/13ff14b732e74380a329a2d11d5fe685, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/b021bd1ba6b249e38e1708cc19bf87f6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/50517c0199da46639ee84a9307962fa8, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/0122aa6d9b604519ab1cfbcbae191aaa, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/71f8a19969f34b3e8a6ed864ca31bb9f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/253d3679bc45456ba2348e080fe576ad] to archive 2024-04-14T11:16:43,397 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:16:43,399 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/abd04d31efdb4bb98d5dda74c65288b0 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/abd04d31efdb4bb98d5dda74c65288b0 2024-04-14T11:16:43,400 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/788dc4c85ec4450fabcb862c7411afc4 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/788dc4c85ec4450fabcb862c7411afc4 2024-04-14T11:16:43,401 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/3cc1f600dbb042b0ac3468a0eeefb702 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/3cc1f600dbb042b0ac3468a0eeefb702 2024-04-14T11:16:43,403 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/9a84a28639f14d8c9f09dcd386c3226c to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/9a84a28639f14d8c9f09dcd386c3226c 2024-04-14T11:16:43,404 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/04e29a382f9b4e57a1de33850a41cc1b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/04e29a382f9b4e57a1de33850a41cc1b 2024-04-14T11:16:43,405 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/d1e35a799d6b4cc486d97ca00857dba6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/d1e35a799d6b4cc486d97ca00857dba6 2024-04-14T11:16:43,407 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/95ab2d9391b044b681fb0aa3d12ba808 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/95ab2d9391b044b681fb0aa3d12ba808 2024-04-14T11:16:43,408 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/a3e8835ba40c4bb394e720fdd0df479c to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/a3e8835ba40c4bb394e720fdd0df479c 2024-04-14T11:16:43,409 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/beb94d3fec3c44edb2e7fd854eeede31 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/beb94d3fec3c44edb2e7fd854eeede31 2024-04-14T11:16:43,411 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f6daee50e0eb453b9e280eb3416e25ed to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f6daee50e0eb453b9e280eb3416e25ed 2024-04-14T11:16:43,412 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/8f8779931cdd404b8c88a8f1cb91eb50 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/8f8779931cdd404b8c88a8f1cb91eb50 2024-04-14T11:16:43,413 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/5d5bd0f8a39b476d970be6b4dd0d5baa to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/5d5bd0f8a39b476d970be6b4dd0d5baa 2024-04-14T11:16:43,414 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f855465c00404beeaae3d59dc8253de3 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f855465c00404beeaae3d59dc8253de3 2024-04-14T11:16:43,416 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/3d7401081d6c452b9ba6c5ab204373f3 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/3d7401081d6c452b9ba6c5ab204373f3 2024-04-14T11:16:43,417 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/a5e26fa4198141a9b3b0d132e45dbd42 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/a5e26fa4198141a9b3b0d132e45dbd42 2024-04-14T11:16:43,418 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f87225c8929942abb75fd4010c2b2ad7 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f87225c8929942abb75fd4010c2b2ad7 2024-04-14T11:16:43,420 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/be340246078e4e65a961087e1428fa5b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/be340246078e4e65a961087e1428fa5b 2024-04-14T11:16:43,421 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/840b2decca8946b1a065decae005847e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/840b2decca8946b1a065decae005847e 2024-04-14T11:16:43,422 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/296996bf35664c14888bde44ccf3d3ad to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/296996bf35664c14888bde44ccf3d3ad 2024-04-14T11:16:43,424 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/2c195e64efb14cff9ca4a7210ccb6821 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/2c195e64efb14cff9ca4a7210ccb6821 2024-04-14T11:16:43,425 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/d03d9de8bde64b5c9c591980dfaec1d5 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/d03d9de8bde64b5c9c591980dfaec1d5 2024-04-14T11:16:43,426 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/b4f8c2822ba24305b9629c2f211fb0e8 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/b4f8c2822ba24305b9629c2f211fb0e8 2024-04-14T11:16:43,427 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/e26d1bdd030b451381ee04da62cc0490 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/e26d1bdd030b451381ee04da62cc0490 2024-04-14T11:16:43,428 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/1534a7d91d9c4352a012a263005b1fcc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/1534a7d91d9c4352a012a263005b1fcc 2024-04-14T11:16:43,429 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/cd1da2f729a044af91ad3f50f63ecd0b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/cd1da2f729a044af91ad3f50f63ecd0b 2024-04-14T11:16:43,430 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/bd30a71ec07b4835be75d98ba18a8d8e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/bd30a71ec07b4835be75d98ba18a8d8e 2024-04-14T11:16:43,431 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/8dd7b20164e94073b31037f685194213 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/8dd7b20164e94073b31037f685194213 2024-04-14T11:16:43,433 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f73d48bd452f4fd0a7b3ce87de82a024 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/f73d48bd452f4fd0a7b3ce87de82a024 2024-04-14T11:16:43,434 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/1e3c7aa35384457da9f4c1e8fb6c8760 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/1e3c7aa35384457da9f4c1e8fb6c8760 2024-04-14T11:16:43,435 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/c0366f52f3e245eda187348a72a77134 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/c0366f52f3e245eda187348a72a77134 2024-04-14T11:16:43,436 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/e37b04c2cbdd40c4ad5555a9f649149a to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/e37b04c2cbdd40c4ad5555a9f649149a 2024-04-14T11:16:43,438 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/ca387b1fb9dd44efba1ffae40531b3b7 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/ca387b1fb9dd44efba1ffae40531b3b7 2024-04-14T11:16:43,439 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/402b9cb8178b46e7935beef58047f2f2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/402b9cb8178b46e7935beef58047f2f2 2024-04-14T11:16:43,440 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/a5c7a1207e304fefa3f4d6f232f5c14b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/a5c7a1207e304fefa3f4d6f232f5c14b 2024-04-14T11:16:43,441 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/13ff14b732e74380a329a2d11d5fe685 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/13ff14b732e74380a329a2d11d5fe685 2024-04-14T11:16:43,443 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/b021bd1ba6b249e38e1708cc19bf87f6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/b021bd1ba6b249e38e1708cc19bf87f6 2024-04-14T11:16:43,444 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/50517c0199da46639ee84a9307962fa8 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/50517c0199da46639ee84a9307962fa8 2024-04-14T11:16:43,445 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/0122aa6d9b604519ab1cfbcbae191aaa to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/0122aa6d9b604519ab1cfbcbae191aaa 2024-04-14T11:16:43,446 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/71f8a19969f34b3e8a6ed864ca31bb9f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/71f8a19969f34b3e8a6ed864ca31bb9f 2024-04-14T11:16:43,447 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/253d3679bc45456ba2348e080fe576ad to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/253d3679bc45456ba2348e080fe576ad 2024-04-14T11:16:43,449 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/33f990f89a6044108dc8f3e9774fa980, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/23c9766ea79844c9a851602294bc32e6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/823c64db2e1e4b1ea7e3d7587de75a78, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/330568a701f14ce18dfacbf17088eaa2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/21cac03151784bb29892257f51888230, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c1f412b242fc48698bd32eaffc9b5300, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/302c612a3a0345aca31d4f6540c791a1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2baa9cc6ae7a42a892b42bcdad7da609, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a027c7702b604ab29b1f8196a7bfa92b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/d0d90505773c48afaa225faf8f0bbacc, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/3ce8feaa080741d6835a0bf425f4c6c6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/b1978d58ff52484fa3cd4a2ca26d7aef, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/ae5cd44b9fa74344b7ad0cd23e7c48f9, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a364dc55e8aa43b68b202b508544d40d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a55c6a042585461e8f2bfb7209e727fc, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a0c6e3ef70b54436b24f95419251f811, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c4a6cd29faac4cd9b80d1a0290509e55, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/ccacfd255932421bbec7550f0a788503, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/862b1ccd63d24ffab3d898d064c49a74, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/0c61e200cba44f9fb8a1690373f06207, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/f9dc4b5855c84031898e39f1adef88a1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/7441c59e40af4bd890a5476a1475758c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a4043a3714c6404aa574e454122ff07f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/287114d8854d491880f1c36eeccc8f1e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2606205d9b4b45f898436804ea8de3e6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/4b1fc854b0224414a2483d8722e3f793, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2568de2b4f8a4c298328e27e0b5a0525, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/25cfb45c22d0412692fd1299adfb67fe, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/743702e628754c8aa1f449fdd6bdd5dc, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c20eecdb953542ce8a4e64db7c2bdda3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/dcbe2fe3dcc94cbfb36405844f1f6bea, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/527fe695f8d44c82baae5b17f54e9386, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/736b854f227f40159ff1c8876e7a3e37, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/991fc56df8254f4a9a4cab76fc3cdc8d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/3edc416d1bb5421aab36b95328a1d059, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c934183050c646d5a3ed71e494392a19, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/7349d7d2ff744625b810f5524172f512, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/d757003730e04cb5bc43a7d4e0508e66, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/f2ad9e36e6f54119bf7572b270878f40, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/43f54c2918a4499f914b24b8b3fea2d5] to archive 2024-04-14T11:16:43,450 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:16:43,452 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/33f990f89a6044108dc8f3e9774fa980 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/33f990f89a6044108dc8f3e9774fa980 2024-04-14T11:16:43,453 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/23c9766ea79844c9a851602294bc32e6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/23c9766ea79844c9a851602294bc32e6 2024-04-14T11:16:43,455 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/823c64db2e1e4b1ea7e3d7587de75a78 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/823c64db2e1e4b1ea7e3d7587de75a78 2024-04-14T11:16:43,456 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/330568a701f14ce18dfacbf17088eaa2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/330568a701f14ce18dfacbf17088eaa2 2024-04-14T11:16:43,457 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/21cac03151784bb29892257f51888230 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/21cac03151784bb29892257f51888230 2024-04-14T11:16:43,458 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c1f412b242fc48698bd32eaffc9b5300 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c1f412b242fc48698bd32eaffc9b5300 2024-04-14T11:16:43,459 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/302c612a3a0345aca31d4f6540c791a1 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/302c612a3a0345aca31d4f6540c791a1 2024-04-14T11:16:43,460 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2baa9cc6ae7a42a892b42bcdad7da609 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2baa9cc6ae7a42a892b42bcdad7da609 2024-04-14T11:16:43,462 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a027c7702b604ab29b1f8196a7bfa92b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a027c7702b604ab29b1f8196a7bfa92b 2024-04-14T11:16:43,463 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/d0d90505773c48afaa225faf8f0bbacc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/d0d90505773c48afaa225faf8f0bbacc 2024-04-14T11:16:43,464 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/3ce8feaa080741d6835a0bf425f4c6c6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/3ce8feaa080741d6835a0bf425f4c6c6 2024-04-14T11:16:43,465 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/b1978d58ff52484fa3cd4a2ca26d7aef to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/b1978d58ff52484fa3cd4a2ca26d7aef 2024-04-14T11:16:43,466 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/ae5cd44b9fa74344b7ad0cd23e7c48f9 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/ae5cd44b9fa74344b7ad0cd23e7c48f9 2024-04-14T11:16:43,468 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a364dc55e8aa43b68b202b508544d40d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a364dc55e8aa43b68b202b508544d40d 2024-04-14T11:16:43,469 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a55c6a042585461e8f2bfb7209e727fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a55c6a042585461e8f2bfb7209e727fc 2024-04-14T11:16:43,470 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a0c6e3ef70b54436b24f95419251f811 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a0c6e3ef70b54436b24f95419251f811 2024-04-14T11:16:43,471 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c4a6cd29faac4cd9b80d1a0290509e55 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c4a6cd29faac4cd9b80d1a0290509e55 2024-04-14T11:16:43,472 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/ccacfd255932421bbec7550f0a788503 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/ccacfd255932421bbec7550f0a788503 2024-04-14T11:16:43,474 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/862b1ccd63d24ffab3d898d064c49a74 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/862b1ccd63d24ffab3d898d064c49a74 2024-04-14T11:16:43,475 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/0c61e200cba44f9fb8a1690373f06207 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/0c61e200cba44f9fb8a1690373f06207 2024-04-14T11:16:43,476 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/f9dc4b5855c84031898e39f1adef88a1 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/f9dc4b5855c84031898e39f1adef88a1 2024-04-14T11:16:43,477 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/7441c59e40af4bd890a5476a1475758c to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/7441c59e40af4bd890a5476a1475758c 2024-04-14T11:16:43,479 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a4043a3714c6404aa574e454122ff07f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/a4043a3714c6404aa574e454122ff07f 2024-04-14T11:16:43,480 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/287114d8854d491880f1c36eeccc8f1e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/287114d8854d491880f1c36eeccc8f1e 2024-04-14T11:16:43,481 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2606205d9b4b45f898436804ea8de3e6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2606205d9b4b45f898436804ea8de3e6 2024-04-14T11:16:43,482 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/4b1fc854b0224414a2483d8722e3f793 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/4b1fc854b0224414a2483d8722e3f793 2024-04-14T11:16:43,483 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2568de2b4f8a4c298328e27e0b5a0525 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2568de2b4f8a4c298328e27e0b5a0525 2024-04-14T11:16:43,484 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/25cfb45c22d0412692fd1299adfb67fe to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/25cfb45c22d0412692fd1299adfb67fe 2024-04-14T11:16:43,486 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/743702e628754c8aa1f449fdd6bdd5dc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/743702e628754c8aa1f449fdd6bdd5dc 2024-04-14T11:16:43,487 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c20eecdb953542ce8a4e64db7c2bdda3 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c20eecdb953542ce8a4e64db7c2bdda3 2024-04-14T11:16:43,488 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/dcbe2fe3dcc94cbfb36405844f1f6bea to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/dcbe2fe3dcc94cbfb36405844f1f6bea 2024-04-14T11:16:43,489 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/527fe695f8d44c82baae5b17f54e9386 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/527fe695f8d44c82baae5b17f54e9386 2024-04-14T11:16:43,490 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/736b854f227f40159ff1c8876e7a3e37 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/736b854f227f40159ff1c8876e7a3e37 2024-04-14T11:16:43,491 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/991fc56df8254f4a9a4cab76fc3cdc8d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/991fc56df8254f4a9a4cab76fc3cdc8d 2024-04-14T11:16:43,492 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/3edc416d1bb5421aab36b95328a1d059 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/3edc416d1bb5421aab36b95328a1d059 2024-04-14T11:16:43,493 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c934183050c646d5a3ed71e494392a19 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/c934183050c646d5a3ed71e494392a19 2024-04-14T11:16:43,495 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/7349d7d2ff744625b810f5524172f512 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/7349d7d2ff744625b810f5524172f512 2024-04-14T11:16:43,496 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/d757003730e04cb5bc43a7d4e0508e66 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/d757003730e04cb5bc43a7d4e0508e66 2024-04-14T11:16:43,497 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/f2ad9e36e6f54119bf7572b270878f40 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/f2ad9e36e6f54119bf7572b270878f40 2024-04-14T11:16:43,498 DEBUG [StoreCloser-TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/43f54c2918a4499f914b24b8b3fea2d5 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/43f54c2918a4499f914b24b8b3fea2d5 2024-04-14T11:16:43,503 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/recovered.edits/601.seqid, newMaxSeqId=601, maxSeqId=1 2024-04-14T11:16:43,504 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1917): Closed TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2. 2024-04-14T11:16:43,504 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1630): Region close journal for c0901aa47578fba1a7f58734a4a0e3b2: 2024-04-14T11:16:43,506 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(163): Closed c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:43,506 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=90 updating hbase:meta row=c0901aa47578fba1a7f58734a4a0e3b2, regionState=CLOSED 2024-04-14T11:16:43,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=91, resume processing ppid=90 2024-04-14T11:16:43,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=91, ppid=90, state=SUCCESS; CloseRegionProcedure c0901aa47578fba1a7f58734a4a0e3b2, server=jenkins-hbase6.apache.org,33239,1713093321027 in 1.5880 sec 2024-04-14T11:16:43,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=90, resume processing ppid=89 2024-04-14T11:16:43,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=90, ppid=89, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0901aa47578fba1a7f58734a4a0e3b2, UNASSIGN in 1.5930 sec 2024-04-14T11:16:43,511 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093403510"}]},"ts":"1713093403510"} 2024-04-14T11:16:43,512 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-04-14T11:16:43,520 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set TestAcidGuarantees to state=DISABLED 2024-04-14T11:16:43,522 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=89, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.6150 sec 2024-04-14T11:16:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2024-04-14T11:16:44,017 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$TableFuture(3746): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 89 completed 2024-04-14T11:16:44,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$5(2494): Client=jenkins//172.31.2.20 delete TestAcidGuarantees 2024-04-14T11:16:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:16:44,022 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:16:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=92 2024-04-14T11:16:44,023 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=92, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:16:44,029 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:44,035 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A, FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B, FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C, FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/recovered.edits] 2024-04-14T11:16:44,042 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/04cb06f0ae48452cbd10deda0f5435bf to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/04cb06f0ae48452cbd10deda0f5435bf 2024-04-14T11:16:44,044 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/51592514965a49019948bdfdb3abf649 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/51592514965a49019948bdfdb3abf649 2024-04-14T11:16:44,047 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5daeb26b09e54714bcc5ddaa30647e86 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/A/5daeb26b09e54714bcc5ddaa30647e86 2024-04-14T11:16:44,052 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/108d47ae36534e2abc8716541a260cf2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/108d47ae36534e2abc8716541a260cf2 2024-04-14T11:16:44,055 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/4ef0d101026746ba9a9ad19776d23656 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/4ef0d101026746ba9a9ad19776d23656 2024-04-14T11:16:44,056 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/6ef693781572480ead07b75ac8e14db9 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/B/6ef693781572480ead07b75ac8e14db9 2024-04-14T11:16:44,060 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/0872c70f417a415cb123aa83ceed654a to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/0872c70f417a415cb123aa83ceed654a 2024-04-14T11:16:44,062 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2721f4980e1d4175a7cf8182f33381f7 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/2721f4980e1d4175a7cf8182f33381f7 2024-04-14T11:16:44,065 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/bcb99aec2e13436886287472848298c2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/C/bcb99aec2e13436886287472848298c2 2024-04-14T11:16:44,069 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/recovered.edits/601.seqid to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2/recovered.edits/601.seqid 2024-04-14T11:16:44,069 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/c0901aa47578fba1a7f58734a4a0e3b2 2024-04-14T11:16:44,070 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(293): Archived TestAcidGuarantees regions 2024-04-14T11:16:44,073 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=92, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:16:44,077 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-04-14T11:16:44,082 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(387): Removing 'TestAcidGuarantees' descriptor. 2024-04-14T11:16:44,083 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=92, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:16:44,083 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(377): Removing 'TestAcidGuarantees' from region states. 2024-04-14T11:16:44,084 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1713093404083"}]},"ts":"9223372036854775807"} 2024-04-14T11:16:44,087 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-04-14T11:16:44,087 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => c0901aa47578fba1a7f58734a4a0e3b2, NAME => 'TestAcidGuarantees,,1713093379228.c0901aa47578fba1a7f58734a4a0e3b2.', STARTKEY => '', ENDKEY => ''}] 2024-04-14T11:16:44,087 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(381): Marking 'TestAcidGuarantees' as deleted. 2024-04-14T11:16:44,088 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1713093404087"}]},"ts":"9223372036854775807"} 2024-04-14T11:16:44,091 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-04-14T11:16:44,093 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(127): Finished pid=92, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:16:44,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=92, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 76 msec 2024-04-14T11:16:44,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=92 2024-04-14T11:16:44,124 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$TableFuture(3746): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 92 completed 2024-04-14T11:16:44,143 INFO [Listener at localhost/34025 {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=223 (was 224), OpenFileDescriptor=527 (was 531), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=536 (was 459) - SystemLoadAverage LEAK? -, ProcessCount=168 (was 168), AvailableMemoryMB=2088 (was 2408) 2024-04-14T11:16:44,160 INFO [Listener at localhost/34025 {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=223, OpenFileDescriptor=527, MaxFileDescriptor=60000, SystemLoadAverage=536, ProcessCount=168, AvailableMemoryMB=2088 2024-04-14T11:16:44,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-04-14T11:16:44,163 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$4(2378): Client=jenkins//172.31.2.20 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-04-14T11:16:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-04-14T11:16:44,166 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(84): pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-04-14T11:16:44,166 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:16:44,167 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 93 2024-04-14T11:16:44,167 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(84): pid=93, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-04-14T11:16:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=93 2024-04-14T11:16:44,183 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7088): creating {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a 2024-04-14T11:16:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=93 2024-04-14T11:16:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=93 2024-04-14T11:16:44,592 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(891): Instantiated TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:16:44,592 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Closing 4974fb0fb1f1e4ab2efd659fb8211d67, disabling compactions & flushes 2024-04-14T11:16:44,592 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1698): Closing region TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:44,592 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1719): Waiting without time limit for close lock on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:44,592 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1786): Acquired close lock on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. after waiting 0 ms 2024-04-14T11:16:44,592 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1796): Updates disabled for region TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:44,593 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1917): Closed TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:44,593 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1630): Region close journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:44,594 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(84): pid=93, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-04-14T11:16:44,594 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1713093404594"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1713093404594"}]},"ts":"1713093404594"} 2024-04-14T11:16:44,595 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-04-14T11:16:44,596 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(84): pid=93, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-04-14T11:16:44,596 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093404596"}]},"ts":"1713093404596"} 2024-04-14T11:16:44,598 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-04-14T11:16:44,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=4974fb0fb1f1e4ab2efd659fb8211d67, ASSIGN}] 2024-04-14T11:16:44,610 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(729): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=4974fb0fb1f1e4ab2efd659fb8211d67, ASSIGN 2024-04-14T11:16:44,611 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4974fb0fb1f1e4ab2efd659fb8211d67, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,33239,1713093321027; forceNewPlan=false, retain=false 2024-04-14T11:16:44,762 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=94 updating hbase:meta row=4974fb0fb1f1e4ab2efd659fb8211d67, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:44,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE; OpenRegionProcedure 4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:16:44,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=93 2024-04-14T11:16:44,916 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:44,919 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:44,919 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7267): Opening region: {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} 2024-04-14T11:16:44,919 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:44,920 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(891): Instantiated TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:16:44,920 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7309): checking encryption for 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:44,920 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7312): checking classloading for 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:44,921 INFO [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:44,923 INFO [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4974fb0fb1f1e4ab2efd659fb8211d67 columnFamilyName A 2024-04-14T11:16:44,923 DEBUG [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:16:44,923 INFO [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] regionserver.HStore(324): Store=4974fb0fb1f1e4ab2efd659fb8211d67/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:16:44,924 INFO [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:44,925 INFO [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4974fb0fb1f1e4ab2efd659fb8211d67 columnFamilyName B 2024-04-14T11:16:44,925 DEBUG [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:16:44,925 INFO [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] regionserver.HStore(324): Store=4974fb0fb1f1e4ab2efd659fb8211d67/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:16:44,925 INFO [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:44,927 INFO [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4974fb0fb1f1e4ab2efd659fb8211d67 columnFamilyName C 2024-04-14T11:16:44,927 DEBUG [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:16:44,927 INFO [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] regionserver.HStore(324): Store=4974fb0fb1f1e4ab2efd659fb8211d67/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:16:44,928 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:44,928 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:44,930 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-04-14T11:16:44,932 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1082): writing seq id for 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:44,935 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-04-14T11:16:44,935 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1099): Opened 4974fb0fb1f1e4ab2efd659fb8211d67; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67893205, jitterRate=0.011687591671943665}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-04-14T11:16:44,937 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(998): Region open journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:44,937 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2593): Post open deploy tasks for TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., pid=95, masterSystemTime=1713093404915 2024-04-14T11:16:44,939 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2620): Finished post open deploy task for TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:44,939 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:44,940 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=94 updating hbase:meta row=4974fb0fb1f1e4ab2efd659fb8211d67, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:44,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=95, resume processing ppid=94 2024-04-14T11:16:44,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=95, ppid=94, state=SUCCESS; OpenRegionProcedure 4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 in 177 msec 2024-04-14T11:16:44,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=94, resume processing ppid=93 2024-04-14T11:16:44,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=94, ppid=93, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=4974fb0fb1f1e4ab2efd659fb8211d67, ASSIGN in 335 msec 2024-04-14T11:16:44,946 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(84): pid=93, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-04-14T11:16:44,947 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093404946"}]},"ts":"1713093404946"} 2024-04-14T11:16:44,948 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-04-14T11:16:44,957 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(84): pid=93, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-04-14T11:16:44,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=93, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 794 msec 2024-04-14T11:16:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=93 2024-04-14T11:16:45,273 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 93 completed 2024-04-14T11:16:45,275 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x73e2e28d to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:45,281 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e84f4ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:45,283 DEBUG [Listener at localhost/34025 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:45,284 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:56896, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:45,285 DEBUG [Listener at localhost/34025 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-04-14T11:16:45,287 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:57050, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-04-14T11:16:45,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-04-14T11:16:45,290 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$14(2787): Client=jenkins//172.31.2.20 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-04-14T11:16:45,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=96, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=TestAcidGuarantees 2024-04-14T11:16:45,706 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-04-14T11:16:45,706 INFO [PEWorker-2 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-04-14T11:16:45,709 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-04-14T11:16:45,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=4974fb0fb1f1e4ab2efd659fb8211d67, REOPEN/MOVE}] 2024-04-14T11:16:45,711 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(729): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=4974fb0fb1f1e4ab2efd659fb8211d67, REOPEN/MOVE 2024-04-14T11:16:45,712 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=98 updating hbase:meta row=4974fb0fb1f1e4ab2efd659fb8211d67, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:45,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE; CloseRegionProcedure 4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:16:45,865 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:45,865 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(119): Close 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:45,866 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Closing 4974fb0fb1f1e4ab2efd659fb8211d67, disabling compactions & flushes 2024-04-14T11:16:45,866 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1698): Closing region TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:45,866 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1719): Waiting without time limit for close lock on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:45,866 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1786): Acquired close lock on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. after waiting 0 ms 2024-04-14T11:16:45,866 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1796): Updates disabled for region TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:45,871 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-04-14T11:16:45,871 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1917): Closed TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:45,872 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1630): Region close journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:45,872 WARN [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionServer(3777): Not adding moved region record: 4974fb0fb1f1e4ab2efd659fb8211d67 to self. 2024-04-14T11:16:45,873 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(163): Closed 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:45,874 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=98 updating hbase:meta row=4974fb0fb1f1e4ab2efd659fb8211d67, regionState=CLOSED 2024-04-14T11:16:45,876 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=99, resume processing ppid=98 2024-04-14T11:16:45,876 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=99, ppid=98, state=SUCCESS; CloseRegionProcedure 4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 in 162 msec 2024-04-14T11:16:45,877 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4974fb0fb1f1e4ab2efd659fb8211d67, REOPEN/MOVE; state=CLOSED, location=jenkins-hbase6.apache.org,33239,1713093321027; forceNewPlan=false, retain=true 2024-04-14T11:16:46,027 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=98 updating hbase:meta row=4974fb0fb1f1e4ab2efd659fb8211d67, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,031 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=100, ppid=98, state=RUNNABLE; OpenRegionProcedure 4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:16:46,183 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,187 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:46,187 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7267): Opening region: {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} 2024-04-14T11:16:46,188 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:46,188 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(891): Instantiated TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:16:46,188 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7309): checking encryption for 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:46,188 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7312): checking classloading for 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:46,189 INFO [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:46,190 INFO [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4974fb0fb1f1e4ab2efd659fb8211d67 columnFamilyName A 2024-04-14T11:16:46,191 DEBUG [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:16:46,192 INFO [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] regionserver.HStore(324): Store=4974fb0fb1f1e4ab2efd659fb8211d67/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:16:46,192 INFO [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:46,193 INFO [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4974fb0fb1f1e4ab2efd659fb8211d67 columnFamilyName B 2024-04-14T11:16:46,193 DEBUG [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:16:46,193 INFO [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] regionserver.HStore(324): Store=4974fb0fb1f1e4ab2efd659fb8211d67/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:16:46,194 INFO [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:46,194 INFO [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4974fb0fb1f1e4ab2efd659fb8211d67 columnFamilyName C 2024-04-14T11:16:46,194 DEBUG [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:16:46,195 INFO [StoreOpener-4974fb0fb1f1e4ab2efd659fb8211d67-1 {}] regionserver.HStore(324): Store=4974fb0fb1f1e4ab2efd659fb8211d67/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:16:46,195 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:46,196 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:46,198 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-04-14T11:16:46,199 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1082): writing seq id for 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:46,200 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1099): Opened 4974fb0fb1f1e4ab2efd659fb8211d67; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74342200, jitterRate=0.10778510570526123}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-04-14T11:16:46,201 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(998): Region open journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:46,201 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2593): Post open deploy tasks for TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., pid=100, masterSystemTime=1713093406183 2024-04-14T11:16:46,203 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2620): Finished post open deploy task for TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:46,203 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:46,203 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=98 updating hbase:meta row=4974fb0fb1f1e4ab2efd659fb8211d67, regionState=OPEN, openSeqNum=5, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=100, resume processing ppid=98 2024-04-14T11:16:46,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=100, ppid=98, state=SUCCESS; OpenRegionProcedure 4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 in 173 msec 2024-04-14T11:16:46,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=98, resume processing ppid=97 2024-04-14T11:16:46,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=98, ppid=97, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=4974fb0fb1f1e4ab2efd659fb8211d67, REOPEN/MOVE in 495 msec 2024-04-14T11:16:46,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=97, resume processing ppid=96 2024-04-14T11:16:46,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=97, ppid=96, state=SUCCESS; ReopenTableRegionsProcedure table=TestAcidGuarantees in 500 msec 2024-04-14T11:16:46,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=96, state=SUCCESS; ModifyTableProcedure table=TestAcidGuarantees in 920 msec 2024-04-14T11:16:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=96 2024-04-14T11:16:46,215 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3de200f3 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:46,220 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59707b48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:46,221 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3c3b257b to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:46,227 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11f35f97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:46,228 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6ce72135 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:46,233 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@912d12f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:46,234 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x57a6d1e9 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:46,240 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29dbca1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:46,241 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2eff186a to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:46,245 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fb7adc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:46,246 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x27f15279 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:46,253 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55da0ac7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:46,254 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5ee89741 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:46,258 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7327747f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:46,259 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x45cd2380 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:46,267 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f05e6e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:46,267 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1740da57 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:46,272 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbd3a1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:46,273 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x13c1fb9c to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:16:46,279 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c219d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:16:46,290 DEBUG [hconnection-0x64ee77b8-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:46,295 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:46,295 DEBUG [hconnection-0x65ec0f72-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:46,295 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:56898, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-04-14T11:16:46,297 DEBUG [hconnection-0x23f37783-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:46,297 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:56902, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:46,297 DEBUG [hconnection-0x7ca6f6ac-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:46,297 DEBUG [hconnection-0x752557cf-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=101 2024-04-14T11:16:46,304 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:56916, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:46,305 DEBUG [hconnection-0x31b7653a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:46,305 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:56942, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:46,306 DEBUG [hconnection-0x3535f75b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:46,307 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:56954, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:46,308 DEBUG [hconnection-0x676580a0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:46,308 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:46,308 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:56932, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:46,310 DEBUG [hconnection-0x74ccc95f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:46,310 DEBUG [hconnection-0x2e7cf350-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:16:46,310 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:56966, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:46,311 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:56982, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:46,315 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:56990, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:46,315 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:56972, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:16:46,315 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:46,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:46,330 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:46,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-04-14T11:16:46,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093466354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093466355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093466355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093466355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093466356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404147eb393b4e14146119e22de001c0171a5_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_0/A:col10/1713093406327/Put/seqid=0 2024-04-14T11:16:46,385 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404147eb393b4e14146119e22de001c0171a5_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404147eb393b4e14146119e22de001c0171a5_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:46,386 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/ccd0516a05904f8e80ee3a13f3c5bd39, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:46,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/ccd0516a05904f8e80ee3a13f3c5bd39 is 175, key is test_row_0/A:col10/1713093406327/Put/seqid=0 2024-04-14T11:16:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=101 2024-04-14T11:16:46,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093466460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093466461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093466461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093466461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093466462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,472 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-04-14T11:16:46,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:46,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:46,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:46,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(53): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:46,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:46,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=101 2024-04-14T11:16:46,625 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-04-14T11:16:46,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:46,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:46,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:46,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(53): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:46,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:46,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093466664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093466664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093466665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093466665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093466665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,778 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-04-14T11:16:46,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:46,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:46,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:46,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(53): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:46,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:46,801 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=19, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/ccd0516a05904f8e80ee3a13f3c5bd39 2024-04-14T11:16:46,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/932d6695b4ba49adbbd19ff87491e1be is 50, key is test_row_0/B:col10/1713093406327/Put/seqid=0 2024-04-14T11:16:46,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=19 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/932d6695b4ba49adbbd19ff87491e1be 2024-04-14T11:16:46,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/34e54770ce504ee894b2c8ccf6369c26 is 50, key is test_row_0/C:col10/1713093406327/Put/seqid=0 2024-04-14T11:16:46,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=101 2024-04-14T11:16:46,931 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-04-14T11:16:46,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:46,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:46,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:46,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(53): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:46,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:46,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:46,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093466977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093466978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093466978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093466980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:46,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:46,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093466981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:47,085 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:47,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-04-14T11:16:47,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:47,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:47,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:47,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(53): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:47,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:47,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:47,238 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:47,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-04-14T11:16:47,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:47,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:47,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:47,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(53): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:47,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:47,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=19 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/34e54770ce504ee894b2c8ccf6369c26 2024-04-14T11:16:47,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/ccd0516a05904f8e80ee3a13f3c5bd39 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/ccd0516a05904f8e80ee3a13f3c5bd39 2024-04-14T11:16:47,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/ccd0516a05904f8e80ee3a13f3c5bd39, entries=200, sequenceid=19, filesize=38.6 K 2024-04-14T11:16:47,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/932d6695b4ba49adbbd19ff87491e1be as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/932d6695b4ba49adbbd19ff87491e1be 2024-04-14T11:16:47,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/932d6695b4ba49adbbd19ff87491e1be, entries=150, sequenceid=19, filesize=11.7 K 2024-04-14T11:16:47,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/34e54770ce504ee894b2c8ccf6369c26 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/34e54770ce504ee894b2c8ccf6369c26 2024-04-14T11:16:47,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/34e54770ce504ee894b2c8ccf6369c26, entries=150, sequenceid=19, filesize=11.7 K 2024-04-14T11:16:47,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 966ms, sequenceid=19, compaction requested=false 2024-04-14T11:16:47,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:47,392 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:47,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-04-14T11:16:47,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:47,394 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-04-14T11:16:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=101 2024-04-14T11:16:47,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414988952a38fd6418f8817bed9872eae23_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_0/A:col10/1713093406354/Put/seqid=0 2024-04-14T11:16:47,438 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414988952a38fd6418f8817bed9872eae23_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414988952a38fd6418f8817bed9872eae23_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:47,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/cd7c22d6430b4d989b1d91b7ad92fc09, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:47,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/cd7c22d6430b4d989b1d91b7ad92fc09 is 175, key is test_row_0/A:col10/1713093406354/Put/seqid=0 2024-04-14T11:16:47,474 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/cd7c22d6430b4d989b1d91b7ad92fc09 2024-04-14T11:16:47,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/1e1272da20b241cb83cbaba6606b9150 is 50, key is test_row_0/B:col10/1713093406354/Put/seqid=0 2024-04-14T11:16:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:47,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:47,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093467503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:47,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093467505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:47,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093467512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:47,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093467512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:47,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093467513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:47,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093467619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:47,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093467619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:47,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093467625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:47,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093467626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:47,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093467626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:47,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093467824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:47,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093467824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:47,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093467827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:47,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093467832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:47,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093467834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:47,912 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/1e1272da20b241cb83cbaba6606b9150 2024-04-14T11:16:47,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/d8604f42fe2a4ffe8b1d3c8309fb4163 is 50, key is test_row_0/C:col10/1713093406354/Put/seqid=0 2024-04-14T11:16:47,927 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/d8604f42fe2a4ffe8b1d3c8309fb4163 2024-04-14T11:16:47,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/cd7c22d6430b4d989b1d91b7ad92fc09 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/cd7c22d6430b4d989b1d91b7ad92fc09 2024-04-14T11:16:47,943 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/cd7c22d6430b4d989b1d91b7ad92fc09, entries=150, sequenceid=41, filesize=30.2 K 2024-04-14T11:16:47,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/1e1272da20b241cb83cbaba6606b9150 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/1e1272da20b241cb83cbaba6606b9150 2024-04-14T11:16:47,954 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/1e1272da20b241cb83cbaba6606b9150, entries=150, sequenceid=41, filesize=11.7 K 2024-04-14T11:16:47,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/d8604f42fe2a4ffe8b1d3c8309fb4163 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/d8604f42fe2a4ffe8b1d3c8309fb4163 2024-04-14T11:16:47,960 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/d8604f42fe2a4ffe8b1d3c8309fb4163, entries=150, sequenceid=41, filesize=11.7 K 2024-04-14T11:16:47,960 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3035): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 566ms, sequenceid=41, compaction requested=false 2024-04-14T11:16:47,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:47,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:47,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=102 2024-04-14T11:16:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=102 2024-04-14T11:16:47,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=102, resume processing ppid=101 2024-04-14T11:16:47,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=102, ppid=101, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6450 sec 2024-04-14T11:16:47,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=101, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.6680 sec 2024-04-14T11:16:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:48,132 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:48,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-04-14T11:16:48,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404144fc7eae2a41b440ba014c5e28cc9b740_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_0/A:col10/1713093407511/Put/seqid=0 2024-04-14T11:16:48,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093468165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093468167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093468173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093468174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093468175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,204 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404144fc7eae2a41b440ba014c5e28cc9b740_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404144fc7eae2a41b440ba014c5e28cc9b740_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:48,205 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/9b2d9b9671394ef8afea7d3fb7fff09c, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:48,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/9b2d9b9671394ef8afea7d3fb7fff09c is 175, key is test_row_0/A:col10/1713093407511/Put/seqid=0 2024-04-14T11:16:48,236 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/9b2d9b9671394ef8afea7d3fb7fff09c 2024-04-14T11:16:48,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/f2c2a63e31ae437ba10549842686e603 is 50, key is test_row_0/B:col10/1713093407511/Put/seqid=0 2024-04-14T11:16:48,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093468276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093468277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/f2c2a63e31ae437ba10549842686e603 2024-04-14T11:16:48,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093468283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093468284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093468284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/15279c1426da4138b481cd74ca45a7af is 50, key is test_row_0/C:col10/1713093407511/Put/seqid=0 2024-04-14T11:16:48,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/15279c1426da4138b481cd74ca45a7af 2024-04-14T11:16:48,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/9b2d9b9671394ef8afea7d3fb7fff09c as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/9b2d9b9671394ef8afea7d3fb7fff09c 2024-04-14T11:16:48,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/9b2d9b9671394ef8afea7d3fb7fff09c, entries=200, sequenceid=56, filesize=38.6 K 2024-04-14T11:16:48,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/f2c2a63e31ae437ba10549842686e603 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/f2c2a63e31ae437ba10549842686e603 2024-04-14T11:16:48,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/f2c2a63e31ae437ba10549842686e603, entries=150, sequenceid=56, filesize=11.7 K 2024-04-14T11:16:48,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/15279c1426da4138b481cd74ca45a7af as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/15279c1426da4138b481cd74ca45a7af 2024-04-14T11:16:48,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/15279c1426da4138b481cd74ca45a7af, entries=150, sequenceid=56, filesize=11.7 K 2024-04-14T11:16:48,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 249ms, sequenceid=56, compaction requested=true 2024-04-14T11:16:48,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:48,381 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:48,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:48,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:48,382 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:48,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:48,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:48,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:48,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:48,384 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110002 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:48,384 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/A is initiating minor compaction (all files) 2024-04-14T11:16:48,384 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/A in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:48,384 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/ccd0516a05904f8e80ee3a13f3c5bd39, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/cd7c22d6430b4d989b1d91b7ad92fc09, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/9b2d9b9671394ef8afea7d3fb7fff09c] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=107.4 K 2024-04-14T11:16:48,384 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:48,384 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/ccd0516a05904f8e80ee3a13f3c5bd39, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/cd7c22d6430b4d989b1d91b7ad92fc09, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/9b2d9b9671394ef8afea7d3fb7fff09c] 2024-04-14T11:16:48,384 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:48,385 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/B is initiating minor compaction (all files) 2024-04-14T11:16:48,385 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/B in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:48,385 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/932d6695b4ba49adbbd19ff87491e1be, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/1e1272da20b241cb83cbaba6606b9150, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/f2c2a63e31ae437ba10549842686e603] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=35.1 K 2024-04-14T11:16:48,385 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting ccd0516a05904f8e80ee3a13f3c5bd39, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1713093406327 2024-04-14T11:16:48,385 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 932d6695b4ba49adbbd19ff87491e1be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1713093406327 2024-04-14T11:16:48,386 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting cd7c22d6430b4d989b1d91b7ad92fc09, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1713093406341 2024-04-14T11:16:48,386 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 1e1272da20b241cb83cbaba6606b9150, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1713093406341 2024-04-14T11:16:48,387 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9b2d9b9671394ef8afea7d3fb7fff09c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1713093407498 2024-04-14T11:16:48,387 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting f2c2a63e31ae437ba10549842686e603, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1713093407509 2024-04-14T11:16:48,404 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:48,406 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#B#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:48,407 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/b22f8445f6ad46b09578386e6884a908 is 50, key is test_row_0/B:col10/1713093407511/Put/seqid=0 2024-04-14T11:16:48,413 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202404146966cdd1404f4087b4a26a7b14fae1b8_4974fb0fb1f1e4ab2efd659fb8211d67 store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:48,415 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202404146966cdd1404f4087b4a26a7b14fae1b8_4974fb0fb1f1e4ab2efd659fb8211d67, store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:48,415 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404146966cdd1404f4087b4a26a7b14fae1b8_4974fb0fb1f1e4ab2efd659fb8211d67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=101 2024-04-14T11:16:48,420 INFO [Thread-1753 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 101 completed 2024-04-14T11:16:48,424 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-04-14T11:16:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=103 2024-04-14T11:16:48,430 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:48,431 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:48,431 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:48,451 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#A#compaction#352 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:48,451 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/c0d3f2970e6a41039be8094606d49057 is 175, key is test_row_0/A:col10/1713093407511/Put/seqid=0 2024-04-14T11:16:48,452 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/b22f8445f6ad46b09578386e6884a908 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/b22f8445f6ad46b09578386e6884a908 2024-04-14T11:16:48,460 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/B of 4974fb0fb1f1e4ab2efd659fb8211d67 into b22f8445f6ad46b09578386e6884a908(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:48,460 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:48,460 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/B, priority=13, startTime=1713093408382; duration=0sec 2024-04-14T11:16:48,460 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:48,460 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:B 2024-04-14T11:16:48,460 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:48,462 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:48,462 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/C is initiating minor compaction (all files) 2024-04-14T11:16:48,462 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/C in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:48,462 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/34e54770ce504ee894b2c8ccf6369c26, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/d8604f42fe2a4ffe8b1d3c8309fb4163, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/15279c1426da4138b481cd74ca45a7af] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=35.1 K 2024-04-14T11:16:48,463 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 34e54770ce504ee894b2c8ccf6369c26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1713093406327 2024-04-14T11:16:48,463 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting d8604f42fe2a4ffe8b1d3c8309fb4163, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1713093406341 2024-04-14T11:16:48,464 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 15279c1426da4138b481cd74ca45a7af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1713093407509 2024-04-14T11:16:48,474 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#C#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:48,474 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/184631ea03e94ecd82fd16cb68608369 is 50, key is test_row_0/C:col10/1713093407511/Put/seqid=0 2024-04-14T11:16:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:48,488 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:48,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-04-14T11:16:48,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414d5b3b925996840f9a57f1fb7b6c28c41_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_0/A:col10/1713093408162/Put/seqid=0 2024-04-14T11:16:48,510 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414d5b3b925996840f9a57f1fb7b6c28c41_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414d5b3b925996840f9a57f1fb7b6c28c41_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:48,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093468500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093468501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093468501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093468505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,519 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/58cc7b53e76d4b67ae104dd337a0943d, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:48,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093468511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/58cc7b53e76d4b67ae104dd337a0943d is 175, key is test_row_0/A:col10/1713093408162/Put/seqid=0 2024-04-14T11:16:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=103 2024-04-14T11:16:48,540 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/58cc7b53e76d4b67ae104dd337a0943d 2024-04-14T11:16:48,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/a83201c2ecf6477ab8135b85c08d7964 is 50, key is test_row_0/B:col10/1713093408162/Put/seqid=0 2024-04-14T11:16:48,583 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-04-14T11:16:48,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:48,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:48,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:48,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(53): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:48,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:48,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093468614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093468615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093468615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093468619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093468621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,738 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-04-14T11:16:48,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:48,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:48,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:48,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(53): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:48,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:48,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:48,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=103 2024-04-14T11:16:48,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093468818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093468820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093468821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093468824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:48,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093468828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,868 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/c0d3f2970e6a41039be8094606d49057 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/c0d3f2970e6a41039be8094606d49057 2024-04-14T11:16:48,875 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/A of 4974fb0fb1f1e4ab2efd659fb8211d67 into c0d3f2970e6a41039be8094606d49057(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:48,875 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:48,875 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/A, priority=13, startTime=1713093408381; duration=0sec 2024-04-14T11:16:48,875 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:48,875 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:A 2024-04-14T11:16:48,888 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/184631ea03e94ecd82fd16cb68608369 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/184631ea03e94ecd82fd16cb68608369 2024-04-14T11:16:48,891 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:48,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-04-14T11:16:48,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:48,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:48,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:48,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(53): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:48,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:48,894 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/C of 4974fb0fb1f1e4ab2efd659fb8211d67 into 184631ea03e94ecd82fd16cb68608369(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:48,894 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:48,894 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/C, priority=13, startTime=1713093408382; duration=0sec 2024-04-14T11:16:48,895 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:48,895 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:C 2024-04-14T11:16:48,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/a83201c2ecf6477ab8135b85c08d7964 2024-04-14T11:16:48,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/20b7212b01904353a744a26742ad93af is 50, key is test_row_0/C:col10/1713093408162/Put/seqid=0 2024-04-14T11:16:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=103 2024-04-14T11:16:49,042 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-04-14T11:16:49,046 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-04-14T11:16:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:49,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(53): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:49,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:49,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/20b7212b01904353a744a26742ad93af 2024-04-14T11:16:49,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/58cc7b53e76d4b67ae104dd337a0943d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/58cc7b53e76d4b67ae104dd337a0943d 2024-04-14T11:16:49,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/58cc7b53e76d4b67ae104dd337a0943d, entries=250, sequenceid=81, filesize=47.0 K 2024-04-14T11:16:49,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/a83201c2ecf6477ab8135b85c08d7964 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/a83201c2ecf6477ab8135b85c08d7964 2024-04-14T11:16:49,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/a83201c2ecf6477ab8135b85c08d7964, entries=150, sequenceid=81, filesize=11.7 K 2024-04-14T11:16:49,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/20b7212b01904353a744a26742ad93af as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/20b7212b01904353a744a26742ad93af 2024-04-14T11:16:49,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/20b7212b01904353a744a26742ad93af, entries=150, sequenceid=81, filesize=11.7 K 2024-04-14T11:16:49,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 598ms, sequenceid=81, compaction requested=false 2024-04-14T11:16:49,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:49,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:49,127 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:49,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-04-14T11:16:49,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041413ff8e04d91b4eaa9494698a46987608_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_0/A:col10/1713093408504/Put/seqid=0 2024-04-14T11:16:49,178 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041413ff8e04d91b4eaa9494698a46987608_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041413ff8e04d91b4eaa9494698a46987608_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:49,180 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/85de372ecdcc425b91d562927d864384, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:49,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/85de372ecdcc425b91d562927d864384 is 175, key is test_row_0/A:col10/1713093408504/Put/seqid=0 2024-04-14T11:16:49,196 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/85de372ecdcc425b91d562927d864384 2024-04-14T11:16:49,200 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-04-14T11:16:49,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:49,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:49,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:49,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(53): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:49,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:49,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:49,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/abc746eee3854b2ea83f7709061fc279 is 50, key is test_row_0/B:col10/1713093408504/Put/seqid=0 2024-04-14T11:16:49,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093469223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093469224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093469230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093469232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093469232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093469334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093469334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093469341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093469343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093469344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,354 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-04-14T11:16:49,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:49,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:49,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:49,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(53): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:49,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:49,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:49,508 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-04-14T11:16:49,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:49,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:49,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:49,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(53): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:49,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=103 2024-04-14T11:16:49,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093469543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093469544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093469549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093469552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093469554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/abc746eee3854b2ea83f7709061fc279 2024-04-14T11:16:49,662 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-04-14T11:16:49,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:49,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:49,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:49,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(53): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:49,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:49,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/dfe2e981a48f48f29810fdb392cc03ec is 50, key is test_row_0/C:col10/1713093408504/Put/seqid=0 2024-04-14T11:16:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:49,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/dfe2e981a48f48f29810fdb392cc03ec 2024-04-14T11:16:49,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/85de372ecdcc425b91d562927d864384 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/85de372ecdcc425b91d562927d864384 2024-04-14T11:16:49,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/85de372ecdcc425b91d562927d864384, entries=250, sequenceid=97, filesize=47.0 K 2024-04-14T11:16:49,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/abc746eee3854b2ea83f7709061fc279 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/abc746eee3854b2ea83f7709061fc279 2024-04-14T11:16:49,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/abc746eee3854b2ea83f7709061fc279, entries=150, sequenceid=97, filesize=11.7 K 2024-04-14T11:16:49,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/dfe2e981a48f48f29810fdb392cc03ec as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/dfe2e981a48f48f29810fdb392cc03ec 2024-04-14T11:16:49,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/dfe2e981a48f48f29810fdb392cc03ec, entries=150, sequenceid=97, filesize=11.7 K 2024-04-14T11:16:49,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=140.89 KB/144270 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 604ms, sequenceid=97, compaction requested=true 2024-04-14T11:16:49,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:49,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:49,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:49,731 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:49,731 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:49,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:49,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:49,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:49,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:49,733 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36055 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:49,733 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127285 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:49,733 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/A is initiating minor compaction (all files) 2024-04-14T11:16:49,733 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/B is initiating minor compaction (all files) 2024-04-14T11:16:49,734 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/B in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:49,734 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/A in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:49,734 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/c0d3f2970e6a41039be8094606d49057, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/58cc7b53e76d4b67ae104dd337a0943d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/85de372ecdcc425b91d562927d864384] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=124.3 K 2024-04-14T11:16:49,734 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/b22f8445f6ad46b09578386e6884a908, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/a83201c2ecf6477ab8135b85c08d7964, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/abc746eee3854b2ea83f7709061fc279] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=35.2 K 2024-04-14T11:16:49,734 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:49,734 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/c0d3f2970e6a41039be8094606d49057, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/58cc7b53e76d4b67ae104dd337a0943d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/85de372ecdcc425b91d562927d864384] 2024-04-14T11:16:49,734 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting b22f8445f6ad46b09578386e6884a908, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1713093407509 2024-04-14T11:16:49,735 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c0d3f2970e6a41039be8094606d49057, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1713093407509 2024-04-14T11:16:49,735 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting a83201c2ecf6477ab8135b85c08d7964, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1713093408162 2024-04-14T11:16:49,735 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 58cc7b53e76d4b67ae104dd337a0943d, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1713093408162 2024-04-14T11:16:49,736 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting abc746eee3854b2ea83f7709061fc279, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1713093408504 2024-04-14T11:16:49,736 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 85de372ecdcc425b91d562927d864384, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1713093408497 2024-04-14T11:16:49,746 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:49,747 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#B#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:49,747 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/34f9b5d15919402cb1dfda51ca6d8c1e is 50, key is test_row_0/B:col10/1713093408504/Put/seqid=0 2024-04-14T11:16:49,765 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20240414e8920fa15b51454ea0829b7bbc9496f2_4974fb0fb1f1e4ab2efd659fb8211d67 store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:49,766 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20240414e8920fa15b51454ea0829b7bbc9496f2_4974fb0fb1f1e4ab2efd659fb8211d67, store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:49,767 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414e8920fa15b51454ea0829b7bbc9496f2_4974fb0fb1f1e4ab2efd659fb8211d67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:49,816 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-04-14T11:16:49,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:49,817 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-04-14T11:16:49,818 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#A#compaction#362 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:49,818 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/c3a4b879ca4b4fa981de030d1b1cdaa0 is 175, key is test_row_0/A:col10/1713093408504/Put/seqid=0 2024-04-14T11:16:49,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414d0c170076e5641b1bc8e2e47712b1133_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_0/A:col10/1713093409180/Put/seqid=0 2024-04-14T11:16:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:49,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:49,869 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/c3a4b879ca4b4fa981de030d1b1cdaa0 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/c3a4b879ca4b4fa981de030d1b1cdaa0 2024-04-14T11:16:49,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093469869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093469871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093469872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093469873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,883 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/A of 4974fb0fb1f1e4ab2efd659fb8211d67 into c3a4b879ca4b4fa981de030d1b1cdaa0(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:49,883 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:49,884 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/A, priority=13, startTime=1713093409731; duration=0sec 2024-04-14T11:16:49,884 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:49,884 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:A 2024-04-14T11:16:49,884 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:49,885 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36055 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:49,885 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/C is initiating minor compaction (all files) 2024-04-14T11:16:49,885 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/C in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:49,885 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/184631ea03e94ecd82fd16cb68608369, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/20b7212b01904353a744a26742ad93af, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/dfe2e981a48f48f29810fdb392cc03ec] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=35.2 K 2024-04-14T11:16:49,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093469875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,886 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 184631ea03e94ecd82fd16cb68608369, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1713093407509 2024-04-14T11:16:49,886 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 20b7212b01904353a744a26742ad93af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1713093408162 2024-04-14T11:16:49,887 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting dfe2e981a48f48f29810fdb392cc03ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1713093408504 2024-04-14T11:16:49,906 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#C#compaction#364 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:49,907 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/4ed34d85a6c84447929c66c2490a86e2 is 50, key is test_row_0/C:col10/1713093408504/Put/seqid=0 2024-04-14T11:16:49,914 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414d0c170076e5641b1bc8e2e47712b1133_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414d0c170076e5641b1bc8e2e47712b1133_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/fd93c11110724cd6ad238732a4fd78f6, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/fd93c11110724cd6ad238732a4fd78f6 is 175, key is test_row_0/A:col10/1713093409180/Put/seqid=0 2024-04-14T11:16:49,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093469983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093469983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093469983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093469984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:49,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093469987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:50,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093470188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:50,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093470188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:50,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093470188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:50,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:50,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093470191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:50,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093470200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:50,218 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/34f9b5d15919402cb1dfda51ca6d8c1e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/34f9b5d15919402cb1dfda51ca6d8c1e 2024-04-14T11:16:50,235 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/B of 4974fb0fb1f1e4ab2efd659fb8211d67 into 34f9b5d15919402cb1dfda51ca6d8c1e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:50,235 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:50,235 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/B, priority=13, startTime=1713093409731; duration=0sec 2024-04-14T11:16:50,235 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:50,235 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:B 2024-04-14T11:16:50,362 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/4ed34d85a6c84447929c66c2490a86e2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/4ed34d85a6c84447929c66c2490a86e2 2024-04-14T11:16:50,369 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/fd93c11110724cd6ad238732a4fd78f6 2024-04-14T11:16:50,369 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/C of 4974fb0fb1f1e4ab2efd659fb8211d67 into 4ed34d85a6c84447929c66c2490a86e2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:50,369 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:50,369 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/C, priority=13, startTime=1713093409732; duration=0sec 2024-04-14T11:16:50,369 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:50,369 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:C 2024-04-14T11:16:50,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/78d94f55bd384a7da5daac033166ae72 is 50, key is test_row_0/B:col10/1713093409180/Put/seqid=0 2024-04-14T11:16:50,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093470494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:50,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093470498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:50,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093470502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:50,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093470504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:50,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:50,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093470504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:50,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=103 2024-04-14T11:16:50,789 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/78d94f55bd384a7da5daac033166ae72 2024-04-14T11:16:50,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/257822f3e263462596d394a568df9265 is 50, key is test_row_0/C:col10/1713093409180/Put/seqid=0 2024-04-14T11:16:51,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:51,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093471001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:51,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:51,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093471006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:51,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093471009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:51,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093471014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:51,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093471015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:51,214 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/257822f3e263462596d394a568df9265 2024-04-14T11:16:51,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/fd93c11110724cd6ad238732a4fd78f6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/fd93c11110724cd6ad238732a4fd78f6 2024-04-14T11:16:51,229 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/fd93c11110724cd6ad238732a4fd78f6, entries=150, sequenceid=121, filesize=30.2 K 2024-04-14T11:16:51,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/78d94f55bd384a7da5daac033166ae72 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/78d94f55bd384a7da5daac033166ae72 2024-04-14T11:16:51,236 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/78d94f55bd384a7da5daac033166ae72, entries=150, sequenceid=121, filesize=11.7 K 2024-04-14T11:16:51,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/257822f3e263462596d394a568df9265 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/257822f3e263462596d394a568df9265 2024-04-14T11:16:51,248 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/257822f3e263462596d394a568df9265, entries=150, sequenceid=121, filesize=11.7 K 2024-04-14T11:16:51,249 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3035): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 1433ms, sequenceid=121, compaction requested=false 2024-04-14T11:16:51,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:51,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:51,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=104 2024-04-14T11:16:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=104 2024-04-14T11:16:51,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=104, resume processing ppid=103 2024-04-14T11:16:51,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=104, ppid=103, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8190 sec 2024-04-14T11:16:51,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=103, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.8300 sec 2024-04-14T11:16:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:52,013 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:52,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-04-14T11:16:52,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414943e5d1f06224f8f82258783e9f33b48_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_0/A:col10/1713093409870/Put/seqid=0 2024-04-14T11:16:52,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093472049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093472050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,064 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414943e5d1f06224f8f82258783e9f33b48_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414943e5d1f06224f8f82258783e9f33b48_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:52,065 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/0e50f6849a954b6898efc68fc8971645, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:52,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/0e50f6849a954b6898efc68fc8971645 is 175, key is test_row_0/A:col10/1713093409870/Put/seqid=0 2024-04-14T11:16:52,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093472060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093472061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093472061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,087 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/0e50f6849a954b6898efc68fc8971645 2024-04-14T11:16:52,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/215fc5e9838f4dd6b580db0b86c117db is 50, key is test_row_0/B:col10/1713093409870/Put/seqid=0 2024-04-14T11:16:52,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/215fc5e9838f4dd6b580db0b86c117db 2024-04-14T11:16:52,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/71b7370292e442e9a97749c984a2c2bf is 50, key is test_row_0/C:col10/1713093409870/Put/seqid=0 2024-04-14T11:16:52,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/71b7370292e442e9a97749c984a2c2bf 2024-04-14T11:16:52,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/0e50f6849a954b6898efc68fc8971645 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/0e50f6849a954b6898efc68fc8971645 2024-04-14T11:16:52,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/0e50f6849a954b6898efc68fc8971645, entries=200, sequenceid=137, filesize=38.8 K 2024-04-14T11:16:52,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/215fc5e9838f4dd6b580db0b86c117db as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/215fc5e9838f4dd6b580db0b86c117db 2024-04-14T11:16:52,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/215fc5e9838f4dd6b580db0b86c117db, entries=150, sequenceid=137, filesize=11.8 K 2024-04-14T11:16:52,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/71b7370292e442e9a97749c984a2c2bf as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/71b7370292e442e9a97749c984a2c2bf 2024-04-14T11:16:52,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/71b7370292e442e9a97749c984a2c2bf, entries=150, sequenceid=137, filesize=11.8 K 2024-04-14T11:16:52,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093472163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 153ms, sequenceid=137, compaction requested=true 2024-04-14T11:16:52,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:52,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:52,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:52,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:52,166 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:52,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:52,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:52,166 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:52,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:52,167 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:52,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-04-14T11:16:52,168 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101764 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:52,168 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/A is initiating minor compaction (all files) 2024-04-14T11:16:52,168 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/A in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:52,168 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/c3a4b879ca4b4fa981de030d1b1cdaa0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/fd93c11110724cd6ad238732a4fd78f6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/0e50f6849a954b6898efc68fc8971645] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=99.4 K 2024-04-14T11:16:52,168 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:52,168 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/c3a4b879ca4b4fa981de030d1b1cdaa0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/fd93c11110724cd6ad238732a4fd78f6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/0e50f6849a954b6898efc68fc8971645] 2024-04-14T11:16:52,169 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36308 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:52,169 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/B is initiating minor compaction (all files) 2024-04-14T11:16:52,169 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/B in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:52,169 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/34f9b5d15919402cb1dfda51ca6d8c1e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/78d94f55bd384a7da5daac033166ae72, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/215fc5e9838f4dd6b580db0b86c117db] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=35.5 K 2024-04-14T11:16:52,169 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c3a4b879ca4b4fa981de030d1b1cdaa0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1713093408504 2024-04-14T11:16:52,170 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 34f9b5d15919402cb1dfda51ca6d8c1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1713093408504 2024-04-14T11:16:52,171 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 78d94f55bd384a7da5daac033166ae72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1713093409180 2024-04-14T11:16:52,172 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting fd93c11110724cd6ad238732a4fd78f6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1713093409180 2024-04-14T11:16:52,173 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 215fc5e9838f4dd6b580db0b86c117db, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1713093409870 2024-04-14T11:16:52,173 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 0e50f6849a954b6898efc68fc8971645, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1713093409866 2024-04-14T11:16:52,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414bb1dbe7016784ec3b2802ec7c8d495bd_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_0/A:col10/1713093412057/Put/seqid=0 2024-04-14T11:16:52,195 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:52,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093472195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093472198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093472200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093472202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,212 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#B#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:52,213 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/737fa11041704b33bd2ebf4582eb854a is 50, key is test_row_0/B:col10/1713093409870/Put/seqid=0 2024-04-14T11:16:52,225 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20240414185f1e6ffb9144eab3702ae41ef5d402_4974fb0fb1f1e4ab2efd659fb8211d67 store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:52,227 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20240414185f1e6ffb9144eab3702ae41ef5d402_4974fb0fb1f1e4ab2efd659fb8211d67, store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:52,227 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414185f1e6ffb9144eab3702ae41ef5d402_4974fb0fb1f1e4ab2efd659fb8211d67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:52,277 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414bb1dbe7016784ec3b2802ec7c8d495bd_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414bb1dbe7016784ec3b2802ec7c8d495bd_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:52,279 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/92c01c982d2f46ac8dc93c85b07ceb2c, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:52,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/92c01c982d2f46ac8dc93c85b07ceb2c is 175, key is test_row_0/A:col10/1713093412057/Put/seqid=0 2024-04-14T11:16:52,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093472304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093472306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093472311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093472315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,363 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/737fa11041704b33bd2ebf4582eb854a as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/737fa11041704b33bd2ebf4582eb854a 2024-04-14T11:16:52,370 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/B of 4974fb0fb1f1e4ab2efd659fb8211d67 into 737fa11041704b33bd2ebf4582eb854a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:52,370 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:52,370 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/B, priority=13, startTime=1713093412166; duration=0sec 2024-04-14T11:16:52,370 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:52,370 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:B 2024-04-14T11:16:52,370 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:16:52,373 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36308 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:16:52,373 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/C is initiating minor compaction (all files) 2024-04-14T11:16:52,373 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/C in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:52,373 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/4ed34d85a6c84447929c66c2490a86e2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/257822f3e263462596d394a568df9265, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/71b7370292e442e9a97749c984a2c2bf] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=35.5 K 2024-04-14T11:16:52,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,374 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 4ed34d85a6c84447929c66c2490a86e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1713093408504 2024-04-14T11:16:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093472367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,374 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 257822f3e263462596d394a568df9265, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1713093409180 2024-04-14T11:16:52,375 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 71b7370292e442e9a97749c984a2c2bf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1713093409870 2024-04-14T11:16:52,403 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#C#compaction#373 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:52,403 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/268c02e972b94c90aaf2027cce0c1036 is 50, key is test_row_0/C:col10/1713093409870/Put/seqid=0 2024-04-14T11:16:52,479 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/268c02e972b94c90aaf2027cce0c1036 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/268c02e972b94c90aaf2027cce0c1036 2024-04-14T11:16:52,491 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/C of 4974fb0fb1f1e4ab2efd659fb8211d67 into 268c02e972b94c90aaf2027cce0c1036(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:52,491 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:52,491 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/C, priority=13, startTime=1713093412166; duration=0sec 2024-04-14T11:16:52,491 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:52,491 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:C 2024-04-14T11:16:52,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093472508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093472515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093472520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093472526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=103 2024-04-14T11:16:52,546 INFO [Thread-1753 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 103 completed 2024-04-14T11:16:52,550 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-04-14T11:16:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=105 2024-04-14T11:16:52,556 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:52,557 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:52,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=105 2024-04-14T11:16:52,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093472677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,710 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-04-14T11:16:52,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:52,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:52,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:52,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(53): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:52,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:52,726 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#A#compaction#371 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:52,727 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/cb3fb859030a442fab208d3475a6e4da is 175, key is test_row_0/A:col10/1713093409870/Put/seqid=0 2024-04-14T11:16:52,746 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/cb3fb859030a442fab208d3475a6e4da as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/cb3fb859030a442fab208d3475a6e4da 2024-04-14T11:16:52,754 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/A of 4974fb0fb1f1e4ab2efd659fb8211d67 into cb3fb859030a442fab208d3475a6e4da(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:52,754 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:52,754 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/A, priority=13, startTime=1713093412166; duration=0sec 2024-04-14T11:16:52,754 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:52,754 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:A 2024-04-14T11:16:52,756 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=162, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/92c01c982d2f46ac8dc93c85b07ceb2c 2024-04-14T11:16:52,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/6058f622d87340aeac8031141529c13d is 50, key is test_row_0/B:col10/1713093412057/Put/seqid=0 2024-04-14T11:16:52,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093472821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093472821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093472824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093472828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=105 2024-04-14T11:16:52,864 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:52,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-04-14T11:16:52,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:52,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:52,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:52,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(53): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:52,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:53,017 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:53,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-04-14T11:16:53,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:53,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:53,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:53,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(53): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:53,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:53,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:53,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=105 2024-04-14T11:16:53,170 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:53,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-04-14T11:16:53,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:53,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:53,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:53,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(53): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:53,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:53,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:53,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093473186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:53,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/6058f622d87340aeac8031141529c13d 2024-04-14T11:16:53,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/dcfed90b4af946ecbffe26f486578313 is 50, key is test_row_0/C:col10/1713093412057/Put/seqid=0 2024-04-14T11:16:53,324 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:53,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-04-14T11:16:53,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:53,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:53,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:53,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(53): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:53,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:53,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:53,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093473329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:53,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093473330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:53,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093473330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:53,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093473332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:53,477 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:53,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-04-14T11:16:53,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:53,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:53,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:53,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(53): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:53,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:53,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:53,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/dcfed90b4af946ecbffe26f486578313 2024-04-14T11:16:53,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/92c01c982d2f46ac8dc93c85b07ceb2c as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/92c01c982d2f46ac8dc93c85b07ceb2c 2024-04-14T11:16:53,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/92c01c982d2f46ac8dc93c85b07ceb2c, entries=200, sequenceid=162, filesize=38.8 K 2024-04-14T11:16:53,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/6058f622d87340aeac8031141529c13d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/6058f622d87340aeac8031141529c13d 2024-04-14T11:16:53,631 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:53,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-04-14T11:16:53,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:53,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:53,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:53,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(53): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:53,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:53,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:53,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/6058f622d87340aeac8031141529c13d, entries=150, sequenceid=162, filesize=11.8 K 2024-04-14T11:16:53,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/dcfed90b4af946ecbffe26f486578313 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/dcfed90b4af946ecbffe26f486578313 2024-04-14T11:16:53,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/dcfed90b4af946ecbffe26f486578313, entries=150, sequenceid=162, filesize=11.8 K 2024-04-14T11:16:53,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 1474ms, sequenceid=162, compaction requested=false 2024-04-14T11:16:53,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=105 2024-04-14T11:16:53,785 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:53,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-04-14T11:16:53,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:53,786 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-04-14T11:16:53,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414ad6e0977c3494ce4bcb0b75b2ac68976_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_0/A:col10/1713093412198/Put/seqid=0 2024-04-14T11:16:53,837 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414ad6e0977c3494ce4bcb0b75b2ac68976_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414ad6e0977c3494ce4bcb0b75b2ac68976_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:53,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/d36ea6e369294aec8b03c399006a45be, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:53,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/d36ea6e369294aec8b03c399006a45be is 175, key is test_row_0/A:col10/1713093412198/Put/seqid=0 2024-04-14T11:16:53,849 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/d36ea6e369294aec8b03c399006a45be 2024-04-14T11:16:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/f765bf9640004041a63cee893b3926a6 is 50, key is test_row_0/B:col10/1713093412198/Put/seqid=0 2024-04-14T11:16:53,891 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/f765bf9640004041a63cee893b3926a6 2024-04-14T11:16:53,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/c6ceef3c544d4139818f38d541215ff8 is 50, key is test_row_0/C:col10/1713093412198/Put/seqid=0 2024-04-14T11:16:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:54,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:54,304 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/c6ceef3c544d4139818f38d541215ff8 2024-04-14T11:16:54,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/d36ea6e369294aec8b03c399006a45be as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/d36ea6e369294aec8b03c399006a45be 2024-04-14T11:16:54,317 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/d36ea6e369294aec8b03c399006a45be, entries=150, sequenceid=176, filesize=30.4 K 2024-04-14T11:16:54,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/f765bf9640004041a63cee893b3926a6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/f765bf9640004041a63cee893b3926a6 2024-04-14T11:16:54,329 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/f765bf9640004041a63cee893b3926a6, entries=150, sequenceid=176, filesize=11.8 K 2024-04-14T11:16:54,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/c6ceef3c544d4139818f38d541215ff8 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/c6ceef3c544d4139818f38d541215ff8 2024-04-14T11:16:54,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093474332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,338 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/c6ceef3c544d4139818f38d541215ff8, entries=150, sequenceid=176, filesize=11.8 K 2024-04-14T11:16:54,340 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3035): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 553ms, sequenceid=176, compaction requested=true 2024-04-14T11:16:54,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:54,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:54,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=106 2024-04-14T11:16:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=106 2024-04-14T11:16:54,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=106, resume processing ppid=105 2024-04-14T11:16:54,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=106, ppid=105, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7850 sec 2024-04-14T11:16:54,346 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=105, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.7950 sec 2024-04-14T11:16:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:54,348 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:16:54,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-04-14T11:16:54,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414f84756467730434a8aefd3c842cd9c17_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_0/A:col10/1713093414303/Put/seqid=0 2024-04-14T11:16:54,375 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414f84756467730434a8aefd3c842cd9c17_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414f84756467730434a8aefd3c842cd9c17_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:54,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093474366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093474367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093474368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093474369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,378 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/d77b15226d144121b67336ebe6c92fed, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:54,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/d77b15226d144121b67336ebe6c92fed is 175, key is test_row_0/A:col10/1713093414303/Put/seqid=0 2024-04-14T11:16:54,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093474436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093474477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093474477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093474477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093474478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093474645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=105 2024-04-14T11:16:54,661 INFO [Thread-1753 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 105 completed 2024-04-14T11:16:54,663 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-04-14T11:16:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=107 2024-04-14T11:16:54,666 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:54,667 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:54,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:54,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093474685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093474685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093474685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093474685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=107 2024-04-14T11:16:54,792 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/d77b15226d144121b67336ebe6c92fed 2024-04-14T11:16:54,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/3ba534c3dfad4dc5b8907b35ec0a105d is 50, key is test_row_0/B:col10/1713093414303/Put/seqid=0 2024-04-14T11:16:54,819 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-04-14T11:16:54,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:54,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:54,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:54,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(53): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:54,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:54,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093474951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=107 2024-04-14T11:16:54,972 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-04-14T11:16:54,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:54,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:54,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:54,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(53): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:54,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:54,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093474990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093474991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093474992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:54,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093474993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:55,126 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:55,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-04-14T11:16:55,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:55,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:55,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:55,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(53): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:55,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:55,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/3ba534c3dfad4dc5b8907b35ec0a105d 2024-04-14T11:16:55,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/d2edc037e88b4d4caba2c5c9bc767005 is 50, key is test_row_0/C:col10/1713093414303/Put/seqid=0 2024-04-14T11:16:55,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/d2edc037e88b4d4caba2c5c9bc767005 2024-04-14T11:16:55,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/d77b15226d144121b67336ebe6c92fed as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/d77b15226d144121b67336ebe6c92fed 2024-04-14T11:16:55,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/d77b15226d144121b67336ebe6c92fed, entries=150, sequenceid=203, filesize=30.4 K 2024-04-14T11:16:55,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/3ba534c3dfad4dc5b8907b35ec0a105d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/3ba534c3dfad4dc5b8907b35ec0a105d 2024-04-14T11:16:55,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=107 2024-04-14T11:16:55,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/3ba534c3dfad4dc5b8907b35ec0a105d, entries=150, sequenceid=203, filesize=11.8 K 2024-04-14T11:16:55,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/d2edc037e88b4d4caba2c5c9bc767005 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/d2edc037e88b4d4caba2c5c9bc767005 2024-04-14T11:16:55,279 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:55,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-04-14T11:16:55,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:55,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:55,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:55,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(53): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:55,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:55,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/d2edc037e88b4d4caba2c5c9bc767005, entries=150, sequenceid=203, filesize=11.8 K 2024-04-14T11:16:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:16:55,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 933ms, sequenceid=203, compaction requested=true 2024-04-14T11:16:55,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:55,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:16:55,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:55,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:16:55,281 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:55,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:55,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:16:55,282 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:55,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:55,285 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133304 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:55,285 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/A is initiating minor compaction (all files) 2024-04-14T11:16:55,285 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:55,285 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/A in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:55,285 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/B is initiating minor compaction (all files) 2024-04-14T11:16:55,285 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/B in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:55,285 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/cb3fb859030a442fab208d3475a6e4da, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/92c01c982d2f46ac8dc93c85b07ceb2c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/d36ea6e369294aec8b03c399006a45be, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/d77b15226d144121b67336ebe6c92fed] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=130.2 K 2024-04-14T11:16:55,285 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/737fa11041704b33bd2ebf4582eb854a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/6058f622d87340aeac8031141529c13d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/f765bf9640004041a63cee893b3926a6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/3ba534c3dfad4dc5b8907b35ec0a105d] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=47.7 K 2024-04-14T11:16:55,285 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:55,285 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/cb3fb859030a442fab208d3475a6e4da, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/92c01c982d2f46ac8dc93c85b07ceb2c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/d36ea6e369294aec8b03c399006a45be, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/d77b15226d144121b67336ebe6c92fed] 2024-04-14T11:16:55,286 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 737fa11041704b33bd2ebf4582eb854a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1713093409870 2024-04-14T11:16:55,286 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting cb3fb859030a442fab208d3475a6e4da, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1713093409870 2024-04-14T11:16:55,287 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 6058f622d87340aeac8031141529c13d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1713093412057 2024-04-14T11:16:55,287 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 92c01c982d2f46ac8dc93c85b07ceb2c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1713093412047 2024-04-14T11:16:55,288 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting d36ea6e369294aec8b03c399006a45be, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1713093412179 2024-04-14T11:16:55,289 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting f765bf9640004041a63cee893b3926a6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1713093412179 2024-04-14T11:16:55,289 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting d77b15226d144121b67336ebe6c92fed, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1713093414303 2024-04-14T11:16:55,289 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 3ba534c3dfad4dc5b8907b35ec0a105d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1713093414303 2024-04-14T11:16:55,303 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:55,304 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#B#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:55,305 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/f0570734c014463f8fb0b6778967f304 is 50, key is test_row_0/B:col10/1713093414303/Put/seqid=0 2024-04-14T11:16:55,305 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20240414194b5c348fb341bbb56ab082875cdf54_4974fb0fb1f1e4ab2efd659fb8211d67 store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:55,307 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20240414194b5c348fb341bbb56ab082875cdf54_4974fb0fb1f1e4ab2efd659fb8211d67, store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:55,307 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414194b5c348fb341bbb56ab082875cdf54_4974fb0fb1f1e4ab2efd659fb8211d67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:55,340 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/f0570734c014463f8fb0b6778967f304 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/f0570734c014463f8fb0b6778967f304 2024-04-14T11:16:55,342 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#A#compaction#383 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:55,343 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/61c084b28c1f4c69926a8379aa3d8bf0 is 175, key is test_row_0/A:col10/1713093414303/Put/seqid=0 2024-04-14T11:16:55,351 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/B of 4974fb0fb1f1e4ab2efd659fb8211d67 into f0570734c014463f8fb0b6778967f304(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:55,351 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:55,351 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/B, priority=12, startTime=1713093415281; duration=0sec 2024-04-14T11:16:55,351 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:16:55,351 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:B 2024-04-14T11:16:55,351 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:16:55,353 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:16:55,353 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/C is initiating minor compaction (all files) 2024-04-14T11:16:55,353 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/C in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:55,353 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/268c02e972b94c90aaf2027cce0c1036, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/dcfed90b4af946ecbffe26f486578313, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/c6ceef3c544d4139818f38d541215ff8, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/d2edc037e88b4d4caba2c5c9bc767005] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=47.7 K 2024-04-14T11:16:55,359 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 268c02e972b94c90aaf2027cce0c1036, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1713093409870 2024-04-14T11:16:55,360 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting dcfed90b4af946ecbffe26f486578313, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1713093412057 2024-04-14T11:16:55,360 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting c6ceef3c544d4139818f38d541215ff8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1713093412179 2024-04-14T11:16:55,361 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting d2edc037e88b4d4caba2c5c9bc767005, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1713093414303 2024-04-14T11:16:55,387 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#C#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:16:55,388 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/0ab35fc5d0c04300b520b3f7db079adf is 50, key is test_row_0/C:col10/1713093414303/Put/seqid=0 2024-04-14T11:16:55,398 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/61c084b28c1f4c69926a8379aa3d8bf0 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/61c084b28c1f4c69926a8379aa3d8bf0 2024-04-14T11:16:55,412 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/A of 4974fb0fb1f1e4ab2efd659fb8211d67 into 61c084b28c1f4c69926a8379aa3d8bf0(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:55,412 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:55,412 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/A, priority=12, startTime=1713093415281; duration=0sec 2024-04-14T11:16:55,412 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:55,412 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:A 2024-04-14T11:16:55,433 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:55,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-04-14T11:16:55,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:55,435 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-04-14T11:16:55,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041448b7c596087e4eeeb768a1e68f5c187a_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_0/A:col10/1713093414350/Put/seqid=0 2024-04-14T11:16:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:55,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:55,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:55,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093475519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093475519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:55,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:55,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093475522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:55,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093475527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:55,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093475527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:55,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093475628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:55,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093475628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:55,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093475628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:55,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093475634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:55,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093475635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=107 2024-04-14T11:16:55,829 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/0ab35fc5d0c04300b520b3f7db079adf as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/0ab35fc5d0c04300b520b3f7db079adf 2024-04-14T11:16:55,835 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/C of 4974fb0fb1f1e4ab2efd659fb8211d67 into 0ab35fc5d0c04300b520b3f7db079adf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:16:55,835 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:55,835 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/C, priority=12, startTime=1713093415281; duration=0sec 2024-04-14T11:16:55,835 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:16:55,835 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:C 2024-04-14T11:16:55,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093475833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:55,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093475833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:55,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093475834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:55,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093475838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:55,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093475838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:55,882 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041448b7c596087e4eeeb768a1e68f5c187a_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041448b7c596087e4eeeb768a1e68f5c187a_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:55,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/2d7ce83491ac4d7094ccf8bf016e48aa, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:55,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/2d7ce83491ac4d7094ccf8bf016e48aa is 175, key is test_row_0/A:col10/1713093414350/Put/seqid=0 2024-04-14T11:16:56,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093476141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:56,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093476141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:56,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093476142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:56,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:56,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093476144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:56,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:56,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093476144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:56,288 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/2d7ce83491ac4d7094ccf8bf016e48aa 2024-04-14T11:16:56,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/570ef17fa5f9477aab480cbe07afc28e is 50, key is test_row_0/B:col10/1713093414350/Put/seqid=0 2024-04-14T11:16:56,302 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/570ef17fa5f9477aab480cbe07afc28e 2024-04-14T11:16:56,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/3b380b5ad1784bae849f18dcc1ccd5c2 is 50, key is test_row_0/C:col10/1713093414350/Put/seqid=0 2024-04-14T11:16:56,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093476645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:56,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093476648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:56,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093476648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:56,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093476652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:56,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093476654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:56,714 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/3b380b5ad1784bae849f18dcc1ccd5c2 2024-04-14T11:16:56,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/2d7ce83491ac4d7094ccf8bf016e48aa as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/2d7ce83491ac4d7094ccf8bf016e48aa 2024-04-14T11:16:56,726 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/2d7ce83491ac4d7094ccf8bf016e48aa, entries=150, sequenceid=214, filesize=30.4 K 2024-04-14T11:16:56,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/570ef17fa5f9477aab480cbe07afc28e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/570ef17fa5f9477aab480cbe07afc28e 2024-04-14T11:16:56,732 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/570ef17fa5f9477aab480cbe07afc28e, entries=150, sequenceid=214, filesize=11.8 K 2024-04-14T11:16:56,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/3b380b5ad1784bae849f18dcc1ccd5c2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/3b380b5ad1784bae849f18dcc1ccd5c2 2024-04-14T11:16:56,738 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/3b380b5ad1784bae849f18dcc1ccd5c2, entries=150, sequenceid=214, filesize=11.8 K 2024-04-14T11:16:56,739 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3035): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 1304ms, sequenceid=214, compaction requested=false 2024-04-14T11:16:56,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:56,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:56,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=108 2024-04-14T11:16:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=108 2024-04-14T11:16:56,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=108, resume processing ppid=107 2024-04-14T11:16:56,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=108, ppid=107, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0730 sec 2024-04-14T11:16:56,743 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=107, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.0790 sec 2024-04-14T11:16:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=107 2024-04-14T11:16:56,772 INFO [Thread-1753 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 107 completed 2024-04-14T11:16:56,774 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-04-14T11:16:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=109 2024-04-14T11:16:56,777 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:56,778 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:56,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=109 2024-04-14T11:16:56,936 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:56,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-04-14T11:16:56,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:56,937 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-04-14T11:16:56,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414c0d58900695c4961b22ab8df9c19ec6a_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_0/A:col10/1713093415525/Put/seqid=0 2024-04-14T11:16:56,987 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414c0d58900695c4961b22ab8df9c19ec6a_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414c0d58900695c4961b22ab8df9c19ec6a_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:56,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/6c6934a93dba4484a4531b88d434cc92, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:56,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/6c6934a93dba4484a4531b88d434cc92 is 175, key is test_row_0/A:col10/1713093415525/Put/seqid=0 2024-04-14T11:16:57,004 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/6c6934a93dba4484a4531b88d434cc92 2024-04-14T11:16:57,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/84edc7ab1ff641c398070c87f0f6f0ac is 50, key is test_row_0/B:col10/1713093415525/Put/seqid=0 2024-04-14T11:16:57,032 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/84edc7ab1ff641c398070c87f0f6f0ac 2024-04-14T11:16:57,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/4f8c8e75c8c8456a9e0f38a09a0c21a3 is 50, key is test_row_0/C:col10/1713093415525/Put/seqid=0 2024-04-14T11:16:57,051 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/4f8c8e75c8c8456a9e0f38a09a0c21a3 2024-04-14T11:16:57,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/6c6934a93dba4484a4531b88d434cc92 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/6c6934a93dba4484a4531b88d434cc92 2024-04-14T11:16:57,064 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/6c6934a93dba4484a4531b88d434cc92, entries=150, sequenceid=242, filesize=30.4 K 2024-04-14T11:16:57,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/84edc7ab1ff641c398070c87f0f6f0ac as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/84edc7ab1ff641c398070c87f0f6f0ac 2024-04-14T11:16:57,071 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/84edc7ab1ff641c398070c87f0f6f0ac, entries=150, sequenceid=242, filesize=11.8 K 2024-04-14T11:16:57,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/4f8c8e75c8c8456a9e0f38a09a0c21a3 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/4f8c8e75c8c8456a9e0f38a09a0c21a3 2024-04-14T11:16:57,079 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/4f8c8e75c8c8456a9e0f38a09a0c21a3, entries=150, sequenceid=242, filesize=11.8 K 2024-04-14T11:16:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=109 2024-04-14T11:16:57,080 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3035): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=0 B/0 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 143ms, sequenceid=242, compaction requested=true 2024-04-14T11:16:57,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:57,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:57,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=110 2024-04-14T11:16:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=110 2024-04-14T11:16:57,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=110, resume processing ppid=109 2024-04-14T11:16:57,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=110, ppid=109, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 303 msec 2024-04-14T11:16:57,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=109, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 309 msec 2024-04-14T11:16:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=109 2024-04-14T11:16:57,381 INFO [Thread-1753 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 109 completed 2024-04-14T11:16:57,383 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-04-14T11:16:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=111 2024-04-14T11:16:57,388 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:57,389 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:57,389 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=111 2024-04-14T11:16:57,541 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:57,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-04-14T11:16:57,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:57,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:57,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:57,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=112 2024-04-14T11:16:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=112 2024-04-14T11:16:57,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=112, resume processing ppid=111 2024-04-14T11:16:57,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=112, ppid=111, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-04-14T11:16:57,548 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=111, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 164 msec 2024-04-14T11:16:57,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=111 2024-04-14T11:16:57,691 INFO [Thread-1753 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 111 completed 2024-04-14T11:16:57,694 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:57,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-04-14T11:16:57,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=113 2024-04-14T11:16:57,698 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:57,698 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:57,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=113 2024-04-14T11:16:57,850 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:57,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-04-14T11:16:57,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:57,851 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-04-14T11:16:57,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414733b6d7a3a2943d385fde6a97e2ff846_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_1/A:col10/1713093417751/Put/seqid=0 2024-04-14T11:16:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:57,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:57,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:57,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093477982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:57,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:57,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093477982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:57,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:57,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093477985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:57,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:57,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093477986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:57,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:57,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093477988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:58,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=113 2024-04-14T11:16:58,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:58,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093478095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:58,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:58,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093478095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:58,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:58,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093478095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:58,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:58,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093478096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:58,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:58,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093478096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:58,269 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414733b6d7a3a2943d385fde6a97e2ff846_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414733b6d7a3a2943d385fde6a97e2ff846_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:58,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/430d38c68b5741f69db957822d932170, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:58,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/430d38c68b5741f69db957822d932170 is 175, key is test_row_1/A:col10/1713093417751/Put/seqid=0 2024-04-14T11:16:58,277 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/430d38c68b5741f69db957822d932170 2024-04-14T11:16:58,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/fb2d6a32b6f544f1b70d966e94b75cfe is 50, key is test_row_1/B:col10/1713093417751/Put/seqid=0 2024-04-14T11:16:58,296 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/fb2d6a32b6f544f1b70d966e94b75cfe 2024-04-14T11:16:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=113 2024-04-14T11:16:58,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093478302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:58,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/0484d10019394b24978894b5a80e389e is 50, key is test_row_1/C:col10/1713093417751/Put/seqid=0 2024-04-14T11:16:58,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:58,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093478303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093478303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:58,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093478303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:58,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093478305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:58,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093478606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:58,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093478611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:58,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093478612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:58,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093478612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:58,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:58,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093478612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:58,715 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/0484d10019394b24978894b5a80e389e 2024-04-14T11:16:58,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/430d38c68b5741f69db957822d932170 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/430d38c68b5741f69db957822d932170 2024-04-14T11:16:58,727 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/430d38c68b5741f69db957822d932170, entries=250, sequenceid=251, filesize=47.2 K 2024-04-14T11:16:58,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/fb2d6a32b6f544f1b70d966e94b75cfe as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/fb2d6a32b6f544f1b70d966e94b75cfe 2024-04-14T11:16:58,734 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/fb2d6a32b6f544f1b70d966e94b75cfe, entries=100, sequenceid=251, filesize=9.5 K 2024-04-14T11:16:58,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/0484d10019394b24978894b5a80e389e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/0484d10019394b24978894b5a80e389e 2024-04-14T11:16:58,740 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/0484d10019394b24978894b5a80e389e, entries=100, sequenceid=251, filesize=9.5 K 2024-04-14T11:16:58,741 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3035): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 890ms, sequenceid=251, compaction requested=true 2024-04-14T11:16:58,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:58,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:58,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=114 2024-04-14T11:16:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=114 2024-04-14T11:16:58,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=114, resume processing ppid=113 2024-04-14T11:16:58,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=114, ppid=113, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0440 sec 2024-04-14T11:16:58,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=113, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.0500 sec 2024-04-14T11:16:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=113 2024-04-14T11:16:58,804 INFO [Thread-1753 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 113 completed 2024-04-14T11:16:58,806 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-04-14T11:16:58,811 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:58,811 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:58,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:16:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=115 2024-04-14T11:16:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=115 2024-04-14T11:16:58,963 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:58,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-04-14T11:16:58,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:58,964 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-04-14T11:16:58,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404142c1a0eaeb86049119ebcd1b0d8074977_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_0/A:col10/1713093417986/Put/seqid=0 2024-04-14T11:16:58,986 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404142c1a0eaeb86049119ebcd1b0d8074977_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404142c1a0eaeb86049119ebcd1b0d8074977_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:58,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/9fcf75e127ab470bad9bce55de3701e1, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:16:58,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/9fcf75e127ab470bad9bce55de3701e1 is 175, key is test_row_0/A:col10/1713093417986/Put/seqid=0 2024-04-14T11:16:59,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=115 2024-04-14T11:16:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:16:59,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:16:59,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:59,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093479121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:59,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:59,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:59,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093479124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:59,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093479124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:59,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:59,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093479125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:59,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:59,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093479126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:59,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093479233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:59,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093479233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:59,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093479233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:59,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093479233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:59,393 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/9fcf75e127ab470bad9bce55de3701e1 2024-04-14T11:16:59,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/88a8c5a3ae294de1b7d0cb0ac7d89103 is 50, key is test_row_0/B:col10/1713093417986/Put/seqid=0 2024-04-14T11:16:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=115 2024-04-14T11:16:59,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093479437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:59,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093479438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:59,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093479438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:59,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093479439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:59,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093479742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:59,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093479744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:59,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:59,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093479744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:59,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:16:59,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093479744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:16:59,807 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/88a8c5a3ae294de1b7d0cb0ac7d89103 2024-04-14T11:16:59,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/95085c15d2514178a6114c1c687f23a2 is 50, key is test_row_0/C:col10/1713093417986/Put/seqid=0 2024-04-14T11:16:59,828 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/95085c15d2514178a6114c1c687f23a2 2024-04-14T11:16:59,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/9fcf75e127ab470bad9bce55de3701e1 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/9fcf75e127ab470bad9bce55de3701e1 2024-04-14T11:16:59,841 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/9fcf75e127ab470bad9bce55de3701e1, entries=150, sequenceid=279, filesize=30.5 K 2024-04-14T11:16:59,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/88a8c5a3ae294de1b7d0cb0ac7d89103 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/88a8c5a3ae294de1b7d0cb0ac7d89103 2024-04-14T11:16:59,849 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/88a8c5a3ae294de1b7d0cb0ac7d89103, entries=150, sequenceid=279, filesize=12.0 K 2024-04-14T11:16:59,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/95085c15d2514178a6114c1c687f23a2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/95085c15d2514178a6114c1c687f23a2 2024-04-14T11:16:59,861 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/95085c15d2514178a6114c1c687f23a2, entries=150, sequenceid=279, filesize=12.0 K 2024-04-14T11:16:59,862 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3035): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 898ms, sequenceid=279, compaction requested=true 2024-04-14T11:16:59,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:16:59,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:16:59,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=116 2024-04-14T11:16:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=116 2024-04-14T11:16:59,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=116, resume processing ppid=115 2024-04-14T11:16:59,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=116, ppid=115, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0520 sec 2024-04-14T11:16:59,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=115, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.0590 sec 2024-04-14T11:16:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=115 2024-04-14T11:16:59,918 INFO [Thread-1753 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 115 completed 2024-04-14T11:16:59,920 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:16:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-04-14T11:16:59,925 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:16:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=117 2024-04-14T11:16:59,925 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:16:59,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=117 2024-04-14T11:17:00,078 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-04-14T11:17:00,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:00,079 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-04-14T11:17:00,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041496ee990144724baea4513091fe7f977e_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_0/A:col10/1713093419122/Put/seqid=0 2024-04-14T11:17:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=117 2024-04-14T11:17:00,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:17:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:00,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093480300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093480301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093480308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093480309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093480310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093480410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093480411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093480417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093480419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093480419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,509 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041496ee990144724baea4513091fe7f977e_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041496ee990144724baea4513091fe7f977e_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:00,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/fd5ae07d9997445186f072956ac6e497, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:00,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/fd5ae07d9997445186f072956ac6e497 is 175, key is test_row_0/A:col10/1713093419122/Put/seqid=0 2024-04-14T11:17:00,519 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/fd5ae07d9997445186f072956ac6e497 2024-04-14T11:17:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=117 2024-04-14T11:17:00,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/b6bca641e33048b1bce1d9a617c5daa2 is 50, key is test_row_0/B:col10/1713093419122/Put/seqid=0 2024-04-14T11:17:00,547 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/b6bca641e33048b1bce1d9a617c5daa2 2024-04-14T11:17:00,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/2773ad32711548df89ddd5e7fa3f7527 is 50, key is test_row_0/C:col10/1713093419122/Put/seqid=0 2024-04-14T11:17:00,576 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/2773ad32711548df89ddd5e7fa3f7527 2024-04-14T11:17:00,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/fd5ae07d9997445186f072956ac6e497 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/fd5ae07d9997445186f072956ac6e497 2024-04-14T11:17:00,590 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/fd5ae07d9997445186f072956ac6e497, entries=100, sequenceid=287, filesize=22.0 K 2024-04-14T11:17:00,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/b6bca641e33048b1bce1d9a617c5daa2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/b6bca641e33048b1bce1d9a617c5daa2 2024-04-14T11:17:00,599 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/b6bca641e33048b1bce1d9a617c5daa2, entries=100, sequenceid=287, filesize=9.6 K 2024-04-14T11:17:00,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/2773ad32711548df89ddd5e7fa3f7527 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/2773ad32711548df89ddd5e7fa3f7527 2024-04-14T11:17:00,607 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/2773ad32711548df89ddd5e7fa3f7527, entries=100, sequenceid=287, filesize=9.6 K 2024-04-14T11:17:00,608 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3035): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=174.43 KB/178620 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 530ms, sequenceid=287, compaction requested=true 2024-04-14T11:17:00,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:17:00,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:00,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=118 2024-04-14T11:17:00,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=118 2024-04-14T11:17:00,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=118, resume processing ppid=117 2024-04-14T11:17:00,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=118, ppid=117, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 683 msec 2024-04-14T11:17:00,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=117, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 693 msec 2024-04-14T11:17:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:00,622 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:00,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-04-14T11:17:00,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414b4a3911124b84bf8b86428cd8d987974_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_0/A:col10/1713093420621/Put/seqid=0 2024-04-14T11:17:00,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093480628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093480628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093480632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093480634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,642 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414b4a3911124b84bf8b86428cd8d987974_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414b4a3911124b84bf8b86428cd8d987974_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:00,643 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/1704bf4b5b1c472db45f39dd60913d02, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:00,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/1704bf4b5b1c472db45f39dd60913d02 is 175, key is test_row_0/A:col10/1713093420621/Put/seqid=0 2024-04-14T11:17:00,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093480635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093480741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093480742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093480747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093480937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093480937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093480951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093480951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:00,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:00,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093480953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=117 2024-04-14T11:17:01,030 INFO [Thread-1753 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 117 completed 2024-04-14T11:17:01,032 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:01,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-04-14T11:17:01,036 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:01,037 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=119 2024-04-14T11:17:01,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:01,060 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/1704bf4b5b1c472db45f39dd60913d02 2024-04-14T11:17:01,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/4f4b4fc41b9847c29f16912795024040 is 50, key is test_row_0/B:col10/1713093420621/Put/seqid=0 2024-04-14T11:17:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=119 2024-04-14T11:17:01,190 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:01,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-04-14T11:17:01,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:01,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:17:01,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:01,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(53): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:01,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:01,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093481258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:01,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093481259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:01,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093481259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:01,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=119 2024-04-14T11:17:01,343 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:01,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-04-14T11:17:01,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:01,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:17:01,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:01,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(53): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:01,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:01,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093481446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:01,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093481448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:01,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/4f4b4fc41b9847c29f16912795024040 2024-04-14T11:17:01,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/d5d07d7c8e6948a1ab0ba0953f7d85b9 is 50, key is test_row_0/C:col10/1713093420621/Put/seqid=0 2024-04-14T11:17:01,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/d5d07d7c8e6948a1ab0ba0953f7d85b9 2024-04-14T11:17:01,496 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:01,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-04-14T11:17:01,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:01,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:17:01,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:01,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(53): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:01,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:01,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/1704bf4b5b1c472db45f39dd60913d02 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/1704bf4b5b1c472db45f39dd60913d02 2024-04-14T11:17:01,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/1704bf4b5b1c472db45f39dd60913d02, entries=200, sequenceid=317, filesize=39.0 K 2024-04-14T11:17:01,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/4f4b4fc41b9847c29f16912795024040 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/4f4b4fc41b9847c29f16912795024040 2024-04-14T11:17:01,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/4f4b4fc41b9847c29f16912795024040, entries=150, sequenceid=317, filesize=12.0 K 2024-04-14T11:17:01,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/d5d07d7c8e6948a1ab0ba0953f7d85b9 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/d5d07d7c8e6948a1ab0ba0953f7d85b9 2024-04-14T11:17:01,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/d5d07d7c8e6948a1ab0ba0953f7d85b9, entries=150, sequenceid=317, filesize=12.0 K 2024-04-14T11:17:01,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=26.84 KB/27480 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 897ms, sequenceid=317, compaction requested=true 2024-04-14T11:17:01,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:17:01,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:01,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:01,519 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-04-14T11:17:01,520 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-04-14T11:17:01,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:01,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:01,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:01,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:01,524 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 235790 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-04-14T11:17:01,524 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/A is initiating minor compaction (all files) 2024-04-14T11:17:01,524 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/A in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:01,524 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 80994 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-04-14T11:17:01,524 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/B is initiating minor compaction (all files) 2024-04-14T11:17:01,524 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/61c084b28c1f4c69926a8379aa3d8bf0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/2d7ce83491ac4d7094ccf8bf016e48aa, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/6c6934a93dba4484a4531b88d434cc92, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/430d38c68b5741f69db957822d932170, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/9fcf75e127ab470bad9bce55de3701e1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/fd5ae07d9997445186f072956ac6e497, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/1704bf4b5b1c472db45f39dd60913d02] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=230.3 K 2024-04-14T11:17:01,524 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/B in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:01,524 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/f0570734c014463f8fb0b6778967f304, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/570ef17fa5f9477aab480cbe07afc28e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/84edc7ab1ff641c398070c87f0f6f0ac, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/fb2d6a32b6f544f1b70d966e94b75cfe, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/88a8c5a3ae294de1b7d0cb0ac7d89103, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/b6bca641e33048b1bce1d9a617c5daa2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/4f4b4fc41b9847c29f16912795024040] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=79.1 K 2024-04-14T11:17:01,524 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:01,524 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/61c084b28c1f4c69926a8379aa3d8bf0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/2d7ce83491ac4d7094ccf8bf016e48aa, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/6c6934a93dba4484a4531b88d434cc92, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/430d38c68b5741f69db957822d932170, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/9fcf75e127ab470bad9bce55de3701e1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/fd5ae07d9997445186f072956ac6e497, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/1704bf4b5b1c472db45f39dd60913d02] 2024-04-14T11:17:01,525 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting f0570734c014463f8fb0b6778967f304, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1713093414303 2024-04-14T11:17:01,525 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 61c084b28c1f4c69926a8379aa3d8bf0, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1713093414303 2024-04-14T11:17:01,525 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 570ef17fa5f9477aab480cbe07afc28e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1713093414350 2024-04-14T11:17:01,525 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2d7ce83491ac4d7094ccf8bf016e48aa, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1713093414350 2024-04-14T11:17:01,526 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 84edc7ab1ff641c398070c87f0f6f0ac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1713093415519 2024-04-14T11:17:01,526 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 6c6934a93dba4484a4531b88d434cc92, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1713093415519 2024-04-14T11:17:01,526 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting fb2d6a32b6f544f1b70d966e94b75cfe, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1713093417750 2024-04-14T11:17:01,527 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 430d38c68b5741f69db957822d932170, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1713093417746 2024-04-14T11:17:01,527 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 88a8c5a3ae294de1b7d0cb0ac7d89103, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1713093417979 2024-04-14T11:17:01,527 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9fcf75e127ab470bad9bce55de3701e1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1713093417979 2024-04-14T11:17:01,528 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting b6bca641e33048b1bce1d9a617c5daa2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1713093419122 2024-04-14T11:17:01,528 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting fd5ae07d9997445186f072956ac6e497, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1713093419122 2024-04-14T11:17:01,528 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 4f4b4fc41b9847c29f16912795024040, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1713093420300 2024-04-14T11:17:01,528 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1704bf4b5b1c472db45f39dd60913d02, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1713093420297 2024-04-14T11:17:01,543 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:01,545 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#B#compaction#404 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:01,546 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024041434667ef7a7a04449b6d170c735593644_4974fb0fb1f1e4ab2efd659fb8211d67 store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:01,546 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/179598e01571418ea8c8cde5a49d840e is 50, key is test_row_0/B:col10/1713093420621/Put/seqid=0 2024-04-14T11:17:01,548 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024041434667ef7a7a04449b6d170c735593644_4974fb0fb1f1e4ab2efd659fb8211d67, store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:01,548 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041434667ef7a7a04449b6d170c735593644_4974fb0fb1f1e4ab2efd659fb8211d67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:01,561 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/179598e01571418ea8c8cde5a49d840e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/179598e01571418ea8c8cde5a49d840e 2024-04-14T11:17:01,567 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 7 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/B of 4974fb0fb1f1e4ab2efd659fb8211d67 into 179598e01571418ea8c8cde5a49d840e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:01,567 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:17:01,567 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/B, priority=9, startTime=1713093421519; duration=0sec 2024-04-14T11:17:01,567 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:01,567 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:B 2024-04-14T11:17:01,567 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-04-14T11:17:01,570 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 80994 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-04-14T11:17:01,570 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/C is initiating minor compaction (all files) 2024-04-14T11:17:01,570 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/C in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:01,570 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/0ab35fc5d0c04300b520b3f7db079adf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/3b380b5ad1784bae849f18dcc1ccd5c2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/4f8c8e75c8c8456a9e0f38a09a0c21a3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/0484d10019394b24978894b5a80e389e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/95085c15d2514178a6114c1c687f23a2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/2773ad32711548df89ddd5e7fa3f7527, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/d5d07d7c8e6948a1ab0ba0953f7d85b9] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=79.1 K 2024-04-14T11:17:01,571 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 0ab35fc5d0c04300b520b3f7db079adf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1713093414303 2024-04-14T11:17:01,571 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 3b380b5ad1784bae849f18dcc1ccd5c2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1713093414350 2024-04-14T11:17:01,571 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 4f8c8e75c8c8456a9e0f38a09a0c21a3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1713093415519 2024-04-14T11:17:01,572 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 0484d10019394b24978894b5a80e389e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1713093417750 2024-04-14T11:17:01,572 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 95085c15d2514178a6114c1c687f23a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1713093417979 2024-04-14T11:17:01,573 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 2773ad32711548df89ddd5e7fa3f7527, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1713093419122 2024-04-14T11:17:01,573 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting d5d07d7c8e6948a1ab0ba0953f7d85b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1713093420300 2024-04-14T11:17:01,586 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#C#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:01,587 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/62e088ae27ec4b68ae3312e60f5e3c3e is 50, key is test_row_0/C:col10/1713093420621/Put/seqid=0 2024-04-14T11:17:01,597 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/62e088ae27ec4b68ae3312e60f5e3c3e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/62e088ae27ec4b68ae3312e60f5e3c3e 2024-04-14T11:17:01,613 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 7 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/C of 4974fb0fb1f1e4ab2efd659fb8211d67 into 62e088ae27ec4b68ae3312e60f5e3c3e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:01,614 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:17:01,614 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/C, priority=9, startTime=1713093421520; duration=0sec 2024-04-14T11:17:01,614 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:01,614 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:C 2024-04-14T11:17:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=119 2024-04-14T11:17:01,649 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:01,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-04-14T11:17:01,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:01,650 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-04-14T11:17:01,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404145cfcff8bb3a4490d95609905d897dcad_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_2/A:col10/1713093420633/Put/seqid=0 2024-04-14T11:17:01,698 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404145cfcff8bb3a4490d95609905d897dcad_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404145cfcff8bb3a4490d95609905d897dcad_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:01,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/8c5b7edc95a64309ae3c4e3f4d68a78b, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:01,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/8c5b7edc95a64309ae3c4e3f4d68a78b is 175, key is test_row_2/A:col10/1713093420633/Put/seqid=0 2024-04-14T11:17:01,725 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/8c5b7edc95a64309ae3c4e3f4d68a78b 2024-04-14T11:17:01,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/9c6a58f3a9d6481ea8223128febf8096 is 50, key is test_row_2/B:col10/1713093420633/Put/seqid=0 2024-04-14T11:17:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:01,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:17:01,960 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#A#compaction#403 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:01,961 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/46a2cef538f1479b8cd2f9725fbf2b47 is 175, key is test_row_0/A:col10/1713093420621/Put/seqid=0 2024-04-14T11:17:01,989 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/46a2cef538f1479b8cd2f9725fbf2b47 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/46a2cef538f1479b8cd2f9725fbf2b47 2024-04-14T11:17:01,997 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 7 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/A of 4974fb0fb1f1e4ab2efd659fb8211d67 into 46a2cef538f1479b8cd2f9725fbf2b47(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:01,997 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:17:01,997 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/A, priority=9, startTime=1713093421519; duration=0sec 2024-04-14T11:17:01,998 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:01,998 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:A 2024-04-14T11:17:02,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093482051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:02,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093482051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:02,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093482053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=119 2024-04-14T11:17:02,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093482155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:02,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093482156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:02,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093482157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:02,173 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/9c6a58f3a9d6481ea8223128febf8096 2024-04-14T11:17:02,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/68f5764ca31e4ce98275c3cbe82aa7d9 is 50, key is test_row_2/C:col10/1713093420633/Put/seqid=0 2024-04-14T11:17:02,186 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/68f5764ca31e4ce98275c3cbe82aa7d9 2024-04-14T11:17:02,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/8c5b7edc95a64309ae3c4e3f4d68a78b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/8c5b7edc95a64309ae3c4e3f4d68a78b 2024-04-14T11:17:02,198 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/8c5b7edc95a64309ae3c4e3f4d68a78b, entries=50, sequenceid=326, filesize=13.5 K 2024-04-14T11:17:02,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/9c6a58f3a9d6481ea8223128febf8096 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/9c6a58f3a9d6481ea8223128febf8096 2024-04-14T11:17:02,206 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/9c6a58f3a9d6481ea8223128febf8096, entries=50, sequenceid=326, filesize=7.2 K 2024-04-14T11:17:02,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/68f5764ca31e4ce98275c3cbe82aa7d9 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/68f5764ca31e4ce98275c3cbe82aa7d9 2024-04-14T11:17:02,213 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/68f5764ca31e4ce98275c3cbe82aa7d9, entries=50, sequenceid=326, filesize=7.2 K 2024-04-14T11:17:02,214 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3035): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 564ms, sequenceid=326, compaction requested=false 2024-04-14T11:17:02,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:17:02,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:02,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=120 2024-04-14T11:17:02,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=120 2024-04-14T11:17:02,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=120, resume processing ppid=119 2024-04-14T11:17:02,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=120, ppid=119, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1770 sec 2024-04-14T11:17:02,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=119, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.1860 sec 2024-04-14T11:17:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:02,364 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:02,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-04-14T11:17:02,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093482366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:02,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093482367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:02,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041422ace97b1e894ba1ab684b3a24cf14b0_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_0/A:col10/1713093422364/Put/seqid=0 2024-04-14T11:17:02,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093482370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:02,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:02,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093482451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:02,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:02,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093482454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:02,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:02,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093482471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:02,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:02,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093482471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:02,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:02,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093482475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:02,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093482673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:02,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093482673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:02,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093482678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:02,781 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041422ace97b1e894ba1ab684b3a24cf14b0_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041422ace97b1e894ba1ab684b3a24cf14b0_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:02,782 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/f77351c32ad44f3ebcb9fe65b2b037fa, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:02,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/f77351c32ad44f3ebcb9fe65b2b037fa is 175, key is test_row_0/A:col10/1713093422364/Put/seqid=0 2024-04-14T11:17:02,788 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=358, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/f77351c32ad44f3ebcb9fe65b2b037fa 2024-04-14T11:17:02,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/e17fa746da5b49c0895723e712ebf65a is 50, key is test_row_0/B:col10/1713093422364/Put/seqid=0 2024-04-14T11:17:02,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/e17fa746da5b49c0895723e712ebf65a 2024-04-14T11:17:02,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/a54982e262c54044b4b363947b37a042 is 50, key is test_row_0/C:col10/1713093422364/Put/seqid=0 2024-04-14T11:17:02,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/a54982e262c54044b4b363947b37a042 2024-04-14T11:17:02,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/f77351c32ad44f3ebcb9fe65b2b037fa as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/f77351c32ad44f3ebcb9fe65b2b037fa 2024-04-14T11:17:02,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/f77351c32ad44f3ebcb9fe65b2b037fa, entries=200, sequenceid=358, filesize=39.0 K 2024-04-14T11:17:02,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/e17fa746da5b49c0895723e712ebf65a as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/e17fa746da5b49c0895723e712ebf65a 2024-04-14T11:17:02,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/e17fa746da5b49c0895723e712ebf65a, entries=150, sequenceid=358, filesize=12.0 K 2024-04-14T11:17:02,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/a54982e262c54044b4b363947b37a042 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/a54982e262c54044b4b363947b37a042 2024-04-14T11:17:02,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/a54982e262c54044b4b363947b37a042, entries=150, sequenceid=358, filesize=12.0 K 2024-04-14T11:17:02,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 485ms, sequenceid=358, compaction requested=true 2024-04-14T11:17:02,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:17:02,849 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:02,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:02,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:02,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:02,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:02,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:02,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-04-14T11:17:02,850 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:02,851 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32648 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:02,852 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/B is initiating minor compaction (all files) 2024-04-14T11:17:02,852 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/B in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:02,852 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/179598e01571418ea8c8cde5a49d840e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/9c6a58f3a9d6481ea8223128febf8096, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/e17fa746da5b49c0895723e712ebf65a] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=31.9 K 2024-04-14T11:17:02,852 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85700 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:02,852 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/A is initiating minor compaction (all files) 2024-04-14T11:17:02,852 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/A in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:02,852 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/46a2cef538f1479b8cd2f9725fbf2b47, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/8c5b7edc95a64309ae3c4e3f4d68a78b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/f77351c32ad44f3ebcb9fe65b2b037fa] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=83.7 K 2024-04-14T11:17:02,852 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:02,852 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/46a2cef538f1479b8cd2f9725fbf2b47, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/8c5b7edc95a64309ae3c4e3f4d68a78b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/f77351c32ad44f3ebcb9fe65b2b037fa] 2024-04-14T11:17:02,854 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 179598e01571418ea8c8cde5a49d840e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1713093420300 2024-04-14T11:17:02,855 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 46a2cef538f1479b8cd2f9725fbf2b47, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1713093420300 2024-04-14T11:17:02,856 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8c5b7edc95a64309ae3c4e3f4d68a78b, keycount=50, bloomtype=ROW, size=13.5 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1713093420633 2024-04-14T11:17:02,857 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting f77351c32ad44f3ebcb9fe65b2b037fa, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1713093422045 2024-04-14T11:17:02,857 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 9c6a58f3a9d6481ea8223128febf8096, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1713093420633 2024-04-14T11:17:02,858 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting e17fa746da5b49c0895723e712ebf65a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1713093422045 2024-04-14T11:17:02,879 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:02,882 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20240414889703cf9b0842f7bc6fe6fa64364016_4974fb0fb1f1e4ab2efd659fb8211d67 store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:02,882 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#B#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:02,883 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/16937dce0804415bb1f8a8d8390b9847 is 50, key is test_row_0/B:col10/1713093422364/Put/seqid=0 2024-04-14T11:17:02,883 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20240414889703cf9b0842f7bc6fe6fa64364016_4974fb0fb1f1e4ab2efd659fb8211d67, store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:02,883 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414889703cf9b0842f7bc6fe6fa64364016_4974fb0fb1f1e4ab2efd659fb8211d67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:02,902 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#A#compaction#412 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:02,903 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/436b0fbb92a34cf0bc969f64e2901b5e is 175, key is test_row_0/A:col10/1713093422364/Put/seqid=0 2024-04-14T11:17:02,916 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/436b0fbb92a34cf0bc969f64e2901b5e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/436b0fbb92a34cf0bc969f64e2901b5e 2024-04-14T11:17:02,922 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/A of 4974fb0fb1f1e4ab2efd659fb8211d67 into 436b0fbb92a34cf0bc969f64e2901b5e(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:02,922 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:17:02,922 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/A, priority=13, startTime=1713093422849; duration=0sec 2024-04-14T11:17:02,922 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:02,922 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:A 2024-04-14T11:17:02,922 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:02,923 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32648 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:02,924 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/C is initiating minor compaction (all files) 2024-04-14T11:17:02,924 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/C in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:02,924 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/62e088ae27ec4b68ae3312e60f5e3c3e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/68f5764ca31e4ce98275c3cbe82aa7d9, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/a54982e262c54044b4b363947b37a042] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=31.9 K 2024-04-14T11:17:02,924 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 62e088ae27ec4b68ae3312e60f5e3c3e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1713093420300 2024-04-14T11:17:02,925 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 68f5764ca31e4ce98275c3cbe82aa7d9, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1713093420633 2024-04-14T11:17:02,925 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting a54982e262c54044b4b363947b37a042, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1713093422045 2024-04-14T11:17:02,932 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#C#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:02,933 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/0efe8e87915742d287db6229e2056d82 is 50, key is test_row_0/C:col10/1713093422364/Put/seqid=0 2024-04-14T11:17:02,943 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/0efe8e87915742d287db6229e2056d82 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/0efe8e87915742d287db6229e2056d82 2024-04-14T11:17:02,949 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/C of 4974fb0fb1f1e4ab2efd659fb8211d67 into 0efe8e87915742d287db6229e2056d82(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:02,949 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:17:02,950 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/C, priority=13, startTime=1713093422850; duration=0sec 2024-04-14T11:17:02,950 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:02,950 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:C 2024-04-14T11:17:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:03,121 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:03,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-04-14T11:17:03,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404147f1d633f8af64034bccfd43244d87af2_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_0/A:col10/1713093423118/Put/seqid=0 2024-04-14T11:17:03,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=119 2024-04-14T11:17:03,144 INFO [Thread-1753 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 119 completed 2024-04-14T11:17:03,144 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404147f1d633f8af64034bccfd43244d87af2_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404147f1d633f8af64034bccfd43244d87af2_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:03,146 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/66b11691e39843efb0ae5fa415ed2390, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:03,146 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:03,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/66b11691e39843efb0ae5fa415ed2390 is 175, key is test_row_0/A:col10/1713093423118/Put/seqid=0 2024-04-14T11:17:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-04-14T11:17:03,150 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=121 2024-04-14T11:17:03,150 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:03,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:03,154 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/66b11691e39843efb0ae5fa415ed2390 2024-04-14T11:17:03,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/9520682c7081496a87ad60f29412e597 is 50, key is test_row_0/B:col10/1713093423118/Put/seqid=0 2024-04-14T11:17:03,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093483182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:03,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093483183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:03,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093483185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=121 2024-04-14T11:17:03,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093483288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:03,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093483288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:03,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093483289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:03,302 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:03,303 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/16937dce0804415bb1f8a8d8390b9847 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/16937dce0804415bb1f8a8d8390b9847 2024-04-14T11:17:03,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-04-14T11:17:03,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:03,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:17:03,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:03,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(53): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:03,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:03,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:03,312 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/B of 4974fb0fb1f1e4ab2efd659fb8211d67 into 16937dce0804415bb1f8a8d8390b9847(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:03,312 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:17:03,312 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/B, priority=13, startTime=1713093422850; duration=0sec 2024-04-14T11:17:03,312 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:03,312 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:B 2024-04-14T11:17:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=121 2024-04-14T11:17:03,455 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:03,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-04-14T11:17:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:17:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:03,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(53): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:03,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:03,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093483491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:03,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093483492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:03,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093483493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:03,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/9520682c7081496a87ad60f29412e597 2024-04-14T11:17:03,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/3f1f955f9efc4a058d6c9c3a3e471cde is 50, key is test_row_0/C:col10/1713093423118/Put/seqid=0 2024-04-14T11:17:03,609 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:03,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-04-14T11:17:03,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:03,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:17:03,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:03,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(53): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:03,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:03,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:03,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=121 2024-04-14T11:17:03,761 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:03,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-04-14T11:17:03,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:03,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:17:03,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:03,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(53): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:03,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:03,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:03,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:03,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093483795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:03,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093483796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:03,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:03,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093483797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:03,915 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:03,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-04-14T11:17:03,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:03,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:17:03,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:03,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(53): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:03,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:03,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/3f1f955f9efc4a058d6c9c3a3e471cde 2024-04-14T11:17:03,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/66b11691e39843efb0ae5fa415ed2390 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/66b11691e39843efb0ae5fa415ed2390 2024-04-14T11:17:03,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/66b11691e39843efb0ae5fa415ed2390, entries=250, sequenceid=371, filesize=47.5 K 2024-04-14T11:17:03,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/9520682c7081496a87ad60f29412e597 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/9520682c7081496a87ad60f29412e597 2024-04-14T11:17:04,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/9520682c7081496a87ad60f29412e597, entries=250, sequenceid=371, filesize=16.8 K 2024-04-14T11:17:04,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/3f1f955f9efc4a058d6c9c3a3e471cde as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/3f1f955f9efc4a058d6c9c3a3e471cde 2024-04-14T11:17:04,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/3f1f955f9efc4a058d6c9c3a3e471cde, entries=150, sequenceid=371, filesize=12.0 K 2024-04-14T11:17:04,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 893ms, sequenceid=371, compaction requested=false 2024-04-14T11:17:04,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:17:04,070 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:04,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-04-14T11:17:04,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:04,071 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-04-14T11:17:04,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404142155929a986242a18696307013ab7a22_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_0/A:col10/1713093423181/Put/seqid=0 2024-04-14T11:17:04,094 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404142155929a986242a18696307013ab7a22_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404142155929a986242a18696307013ab7a22_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:04,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/52bc7fc385f844c0a85ecafd47189e6b, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:04,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/52bc7fc385f844c0a85ecafd47189e6b is 175, key is test_row_0/A:col10/1713093423181/Put/seqid=0 2024-04-14T11:17:04,102 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=397, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/52bc7fc385f844c0a85ecafd47189e6b 2024-04-14T11:17:04,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/c31070ea6cab4e46a14b58c5139ea96e is 50, key is test_row_0/B:col10/1713093423181/Put/seqid=0 2024-04-14T11:17:04,120 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/c31070ea6cab4e46a14b58c5139ea96e 2024-04-14T11:17:04,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/b5f302695c9147e2adcfedda3b1bfb2f is 50, key is test_row_0/C:col10/1713093423181/Put/seqid=0 2024-04-14T11:17:04,134 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/b5f302695c9147e2adcfedda3b1bfb2f 2024-04-14T11:17:04,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/52bc7fc385f844c0a85ecafd47189e6b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/52bc7fc385f844c0a85ecafd47189e6b 2024-04-14T11:17:04,146 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/52bc7fc385f844c0a85ecafd47189e6b, entries=150, sequenceid=397, filesize=30.5 K 2024-04-14T11:17:04,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/c31070ea6cab4e46a14b58c5139ea96e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/c31070ea6cab4e46a14b58c5139ea96e 2024-04-14T11:17:04,152 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/c31070ea6cab4e46a14b58c5139ea96e, entries=150, sequenceid=397, filesize=12.0 K 2024-04-14T11:17:04,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/b5f302695c9147e2adcfedda3b1bfb2f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/b5f302695c9147e2adcfedda3b1bfb2f 2024-04-14T11:17:04,160 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/b5f302695c9147e2adcfedda3b1bfb2f, entries=150, sequenceid=397, filesize=12.0 K 2024-04-14T11:17:04,161 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3035): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 91ms, sequenceid=397, compaction requested=true 2024-04-14T11:17:04,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:17:04,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:04,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=122 2024-04-14T11:17:04,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=122 2024-04-14T11:17:04,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=122, resume processing ppid=121 2024-04-14T11:17:04,166 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=122, ppid=121, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0130 sec 2024-04-14T11:17:04,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=121, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.0200 sec 2024-04-14T11:17:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=121 2024-04-14T11:17:04,255 INFO [Thread-1753 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 121 completed 2024-04-14T11:17:04,257 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-04-14T11:17:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=123 2024-04-14T11:17:04,262 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:04,262 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:04,263 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:04,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=123 2024-04-14T11:17:04,414 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:04,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-04-14T11:17:04,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:04,415 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-04-14T11:17:04,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404145261ec0e4f5f418c97bd269474c2b2ad_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_1/A:col10/1713093424399/Put/seqid=0 2024-04-14T11:17:04,433 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404145261ec0e4f5f418c97bd269474c2b2ad_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404145261ec0e4f5f418c97bd269474c2b2ad_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:04,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/52942831a4fa4f02840df6ec324c5332, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:04,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/52942831a4fa4f02840df6ec324c5332 is 175, key is test_row_1/A:col10/1713093424399/Put/seqid=0 2024-04-14T11:17:04,440 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=403, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/52942831a4fa4f02840df6ec324c5332 2024-04-14T11:17:04,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/8bb35e5254274609a5f9cba4f4d41539 is 50, key is test_row_1/B:col10/1713093424399/Put/seqid=0 2024-04-14T11:17:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:04,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:17:04,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093484536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:04,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093484537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:04,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093484538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:04,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093484539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:04,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093484543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=123 2024-04-14T11:17:04,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093484646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:04,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093484646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:04,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093484646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:04,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093484646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:04,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093484652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:04,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:04,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093484852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:04,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:04,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093484852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:04,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:04,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093484852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:04,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:04,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093484852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:04,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:04,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093484862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:04,863 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/8bb35e5254274609a5f9cba4f4d41539 2024-04-14T11:17:04,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=123 2024-04-14T11:17:04,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/a1c7386c88a04248b32c9addf6e19028 is 50, key is test_row_1/C:col10/1713093424399/Put/seqid=0 2024-04-14T11:17:04,892 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/a1c7386c88a04248b32c9addf6e19028 2024-04-14T11:17:04,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/52942831a4fa4f02840df6ec324c5332 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/52942831a4fa4f02840df6ec324c5332 2024-04-14T11:17:04,903 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/52942831a4fa4f02840df6ec324c5332, entries=150, sequenceid=403, filesize=30.5 K 2024-04-14T11:17:04,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/8bb35e5254274609a5f9cba4f4d41539 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/8bb35e5254274609a5f9cba4f4d41539 2024-04-14T11:17:04,910 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/8bb35e5254274609a5f9cba4f4d41539, entries=150, sequenceid=403, filesize=12.0 K 2024-04-14T11:17:04,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/a1c7386c88a04248b32c9addf6e19028 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/a1c7386c88a04248b32c9addf6e19028 2024-04-14T11:17:04,916 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/a1c7386c88a04248b32c9addf6e19028, entries=100, sequenceid=403, filesize=9.6 K 2024-04-14T11:17:04,917 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3035): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 502ms, sequenceid=403, compaction requested=true 2024-04-14T11:17:04,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:17:04,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:04,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=124 2024-04-14T11:17:04,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=124 2024-04-14T11:17:04,919 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=124, resume processing ppid=123 2024-04-14T11:17:04,919 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=124, ppid=123, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 655 msec 2024-04-14T11:17:04,921 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=123, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 663 msec 2024-04-14T11:17:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:05,159 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:05,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-04-14T11:17:05,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414e78a4712882d4f8bbf36394ad46bec64_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_0/A:col10/1713093425158/Put/seqid=0 2024-04-14T11:17:05,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093485161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093485167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093485168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093485169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,177 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414e78a4712882d4f8bbf36394ad46bec64_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414e78a4712882d4f8bbf36394ad46bec64_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:05,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093485170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,178 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/19d6335104754218a24fe7dd027aa9a0, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:05,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/19d6335104754218a24fe7dd027aa9a0 is 175, key is test_row_0/A:col10/1713093425158/Put/seqid=0 2024-04-14T11:17:05,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093485276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093485277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093485279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=123 2024-04-14T11:17:05,367 INFO [Thread-1753 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 123 completed 2024-04-14T11:17:05,369 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-04-14T11:17:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=125 2024-04-14T11:17:05,372 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:05,373 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:05,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=125 2024-04-14T11:17:05,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093485479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093485482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:05,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093485487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,525 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-04-14T11:17:05,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:05,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:17:05,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:05,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(53): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:05,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:05,584 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=434, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/19d6335104754218a24fe7dd027aa9a0 2024-04-14T11:17:05,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/83513c623ea54416858c1d25f51682ce is 50, key is test_row_0/B:col10/1713093425158/Put/seqid=0 2024-04-14T11:17:05,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/83513c623ea54416858c1d25f51682ce 2024-04-14T11:17:05,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/096c0ce2bbf54e11b6fa7f90d15c8150 is 50, key is test_row_0/C:col10/1713093425158/Put/seqid=0 2024-04-14T11:17:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=125 2024-04-14T11:17:05,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56932 deadline: 1713093485673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56982 deadline: 1713093485678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,680 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-04-14T11:17:05,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:05,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:17:05,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:05,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(53): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:05,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:05,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56916 deadline: 1713093485787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56954 deadline: 1713093485788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:56972 deadline: 1713093485797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,833 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-04-14T11:17:05,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:05,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:17:05,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:05,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(53): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:05,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=125 2024-04-14T11:17:05,986 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:05,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-04-14T11:17:05,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:05,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. as already flushing 2024-04-14T11:17:05,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:05,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(53): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:05,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:06,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/096c0ce2bbf54e11b6fa7f90d15c8150 2024-04-14T11:17:06,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/19d6335104754218a24fe7dd027aa9a0 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/19d6335104754218a24fe7dd027aa9a0 2024-04-14T11:17:06,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/19d6335104754218a24fe7dd027aa9a0, entries=200, sequenceid=434, filesize=39.0 K 2024-04-14T11:17:06,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/83513c623ea54416858c1d25f51682ce as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/83513c623ea54416858c1d25f51682ce 2024-04-14T11:17:06,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/83513c623ea54416858c1d25f51682ce, entries=150, sequenceid=434, filesize=12.0 K 2024-04-14T11:17:06,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/096c0ce2bbf54e11b6fa7f90d15c8150 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/096c0ce2bbf54e11b6fa7f90d15c8150 2024-04-14T11:17:06,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/096c0ce2bbf54e11b6fa7f90d15c8150, entries=150, sequenceid=434, filesize=12.0 K 2024-04-14T11:17:06,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=20.13 KB/20610 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 878ms, sequenceid=434, compaction requested=true 2024-04-14T11:17:06,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:17:06,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:06,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:06,039 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-04-14T11:17:06,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:06,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:06,039 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-04-14T11:17:06,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4974fb0fb1f1e4ab2efd659fb8211d67:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:06,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:06,041 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 183048 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-04-14T11:17:06,041 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/A is initiating minor compaction (all files) 2024-04-14T11:17:06,041 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 67080 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-04-14T11:17:06,041 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/A in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:06,041 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/B is initiating minor compaction (all files) 2024-04-14T11:17:06,041 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/B in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:06,041 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/436b0fbb92a34cf0bc969f64e2901b5e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/66b11691e39843efb0ae5fa415ed2390, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/52bc7fc385f844c0a85ecafd47189e6b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/52942831a4fa4f02840df6ec324c5332, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/19d6335104754218a24fe7dd027aa9a0] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=178.8 K 2024-04-14T11:17:06,041 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/16937dce0804415bb1f8a8d8390b9847, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/9520682c7081496a87ad60f29412e597, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/c31070ea6cab4e46a14b58c5139ea96e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/8bb35e5254274609a5f9cba4f4d41539, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/83513c623ea54416858c1d25f51682ce] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=65.5 K 2024-04-14T11:17:06,041 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:06,041 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/436b0fbb92a34cf0bc969f64e2901b5e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/66b11691e39843efb0ae5fa415ed2390, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/52bc7fc385f844c0a85ecafd47189e6b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/52942831a4fa4f02840df6ec324c5332, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/19d6335104754218a24fe7dd027aa9a0] 2024-04-14T11:17:06,041 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 16937dce0804415bb1f8a8d8390b9847, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1713093422045 2024-04-14T11:17:06,042 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 436b0fbb92a34cf0bc969f64e2901b5e, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1713093422045 2024-04-14T11:17:06,042 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 9520682c7081496a87ad60f29412e597, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1713093423028 2024-04-14T11:17:06,042 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 66b11691e39843efb0ae5fa415ed2390, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1713093423028 2024-04-14T11:17:06,042 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting c31070ea6cab4e46a14b58c5139ea96e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1713093423168 2024-04-14T11:17:06,042 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 52bc7fc385f844c0a85ecafd47189e6b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1713093423168 2024-04-14T11:17:06,043 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 8bb35e5254274609a5f9cba4f4d41539, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1713093424398 2024-04-14T11:17:06,043 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 52942831a4fa4f02840df6ec324c5332, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1713093424398 2024-04-14T11:17:06,043 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 83513c623ea54416858c1d25f51682ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1713093424527 2024-04-14T11:17:06,044 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 19d6335104754218a24fe7dd027aa9a0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1713093424527 2024-04-14T11:17:06,053 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:06,055 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20240414025c812c83c941b1b107ff83d9bc4836_4974fb0fb1f1e4ab2efd659fb8211d67 store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:06,055 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#B#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:06,056 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/1fc5b3436de5403f93558e7f77ded686 is 50, key is test_row_0/B:col10/1713093425158/Put/seqid=0 2024-04-14T11:17:06,057 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20240414025c812c83c941b1b107ff83d9bc4836_4974fb0fb1f1e4ab2efd659fb8211d67, store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:06,057 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414025c812c83c941b1b107ff83d9bc4836_4974fb0fb1f1e4ab2efd659fb8211d67 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:06,069 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#A#compaction#427 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:06,070 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/7a299da7cbcc4b2bb6a012c7037f8864 is 175, key is test_row_0/A:col10/1713093425158/Put/seqid=0 2024-04-14T11:17:06,072 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/1fc5b3436de5403f93558e7f77ded686 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/1fc5b3436de5403f93558e7f77ded686 2024-04-14T11:17:06,078 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 5 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/B of 4974fb0fb1f1e4ab2efd659fb8211d67 into 1fc5b3436de5403f93558e7f77ded686(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:06,078 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:17:06,078 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/B, priority=11, startTime=1713093426039; duration=0sec 2024-04-14T11:17:06,079 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:06,079 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:B 2024-04-14T11:17:06,079 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-04-14T11:17:06,081 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/7a299da7cbcc4b2bb6a012c7037f8864 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/7a299da7cbcc4b2bb6a012c7037f8864 2024-04-14T11:17:06,081 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59760 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-04-14T11:17:06,081 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 4974fb0fb1f1e4ab2efd659fb8211d67/C is initiating minor compaction (all files) 2024-04-14T11:17:06,081 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 4974fb0fb1f1e4ab2efd659fb8211d67/C in TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:06,081 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/0efe8e87915742d287db6229e2056d82, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/3f1f955f9efc4a058d6c9c3a3e471cde, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/b5f302695c9147e2adcfedda3b1bfb2f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/a1c7386c88a04248b32c9addf6e19028, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/096c0ce2bbf54e11b6fa7f90d15c8150] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp, totalSize=58.4 K 2024-04-14T11:17:06,081 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 0efe8e87915742d287db6229e2056d82, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1713093422045 2024-04-14T11:17:06,082 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 3f1f955f9efc4a058d6c9c3a3e471cde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1713093423029 2024-04-14T11:17:06,083 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting b5f302695c9147e2adcfedda3b1bfb2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1713093423168 2024-04-14T11:17:06,083 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting a1c7386c88a04248b32c9addf6e19028, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1713093424399 2024-04-14T11:17:06,084 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 096c0ce2bbf54e11b6fa7f90d15c8150, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1713093424527 2024-04-14T11:17:06,087 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 5 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/A of 4974fb0fb1f1e4ab2efd659fb8211d67 into 7a299da7cbcc4b2bb6a012c7037f8864(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:06,088 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:17:06,088 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/A, priority=11, startTime=1713093426038; duration=0sec 2024-04-14T11:17:06,088 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:06,088 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:A 2024-04-14T11:17:06,094 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4974fb0fb1f1e4ab2efd659fb8211d67#C#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:06,095 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/9f20ec157ab74be7b71cf6f51edad771 is 50, key is test_row_0/C:col10/1713093425158/Put/seqid=0 2024-04-14T11:17:06,140 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:06,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-04-14T11:17:06,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:06,150 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-04-14T11:17:06,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414b0aaf40592b2479db06740561574c6d4_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_1/A:col10/1713093425167/Put/seqid=0 2024-04-14T11:17:06,180 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414b0aaf40592b2479db06740561574c6d4_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414b0aaf40592b2479db06740561574c6d4_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:06,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/d50e706111d843c89aaf5a5495be950a, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:06,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/d50e706111d843c89aaf5a5495be950a is 175, key is test_row_1/A:col10/1713093425167/Put/seqid=0 2024-04-14T11:17:06,212 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=442, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/d50e706111d843c89aaf5a5495be950a 2024-04-14T11:17:06,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/5900360e543b418d9524b7e0590255a8 is 50, key is test_row_1/B:col10/1713093425167/Put/seqid=0 2024-04-14T11:17:06,227 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/5900360e543b418d9524b7e0590255a8 2024-04-14T11:17:06,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/8dde4394c8cd4bbfb94982a18d2bcbe2 is 50, key is test_row_1/C:col10/1713093425167/Put/seqid=0 2024-04-14T11:17:06,310 DEBUG [Thread-1758 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x45cd2380 to 127.0.0.1:60913 2024-04-14T11:17:06,310 DEBUG [Thread-1758 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:06,327 DEBUG [Thread-1754 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x27f15279 to 127.0.0.1:60913 2024-04-14T11:17:06,327 DEBUG [Thread-1754 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:06,329 DEBUG [Thread-1756 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5ee89741 to 127.0.0.1:60913 2024-04-14T11:17:06,329 DEBUG [Thread-1756 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:06,330 DEBUG [Thread-1745 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3c3b257b to 127.0.0.1:60913 2024-04-14T11:17:06,330 DEBUG [Thread-1745 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:06,333 DEBUG [Thread-1747 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6ce72135 to 127.0.0.1:60913 2024-04-14T11:17:06,333 DEBUG [Thread-1747 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:06,334 DEBUG [Thread-1743 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3de200f3 to 127.0.0.1:60913 2024-04-14T11:17:06,334 DEBUG [Thread-1743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:06,356 DEBUG [Thread-1760 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1740da57 to 127.0.0.1:60913 2024-04-14T11:17:06,357 DEBUG [Thread-1760 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:06,373 DEBUG [Thread-1762 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x13c1fb9c to 127.0.0.1:60913 2024-04-14T11:17:06,373 DEBUG [Thread-1762 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:06,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=125 2024-04-14T11:17:06,505 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/9f20ec157ab74be7b71cf6f51edad771 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/9f20ec157ab74be7b71cf6f51edad771 2024-04-14T11:17:06,510 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 5 (all) file(s) in 4974fb0fb1f1e4ab2efd659fb8211d67/C of 4974fb0fb1f1e4ab2efd659fb8211d67 into 9f20ec157ab74be7b71cf6f51edad771(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:06,511 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:17:06,511 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67., storeName=4974fb0fb1f1e4ab2efd659fb8211d67/C, priority=11, startTime=1713093426039; duration=0sec 2024-04-14T11:17:06,511 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:06,511 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4974fb0fb1f1e4ab2efd659fb8211d67:C 2024-04-14T11:17:06,647 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/8dde4394c8cd4bbfb94982a18d2bcbe2 2024-04-14T11:17:06,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/d50e706111d843c89aaf5a5495be950a as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/d50e706111d843c89aaf5a5495be950a 2024-04-14T11:17:06,657 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/d50e706111d843c89aaf5a5495be950a, entries=100, sequenceid=442, filesize=22.0 K 2024-04-14T11:17:06,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/5900360e543b418d9524b7e0590255a8 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/5900360e543b418d9524b7e0590255a8 2024-04-14T11:17:06,663 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/5900360e543b418d9524b7e0590255a8, entries=100, sequenceid=442, filesize=9.6 K 2024-04-14T11:17:06,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/8dde4394c8cd4bbfb94982a18d2bcbe2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/8dde4394c8cd4bbfb94982a18d2bcbe2 2024-04-14T11:17:06,668 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/8dde4394c8cd4bbfb94982a18d2bcbe2, entries=100, sequenceid=442, filesize=9.6 K 2024-04-14T11:17:06,669 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3035): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=20.13 KB/20610 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 519ms, sequenceid=442, compaction requested=false 2024-04-14T11:17:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2533): Flush status journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:17:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=126 2024-04-14T11:17:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=126 2024-04-14T11:17:06,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=126, resume processing ppid=125 2024-04-14T11:17:06,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=126, ppid=125, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2970 sec 2024-04-14T11:17:06,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=125, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.3020 sec 2024-04-14T11:17:06,680 DEBUG [Thread-1751 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2eff186a to 127.0.0.1:60913 2024-04-14T11:17:06,680 DEBUG [Thread-1751 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:06,682 DEBUG [Thread-1749 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x57a6d1e9 to 127.0.0.1:60913 2024-04-14T11:17:06,682 DEBUG [Thread-1749 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=125 2024-04-14T11:17:07,479 INFO [Thread-1753 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 125 completed 2024-04-14T11:17:07,480 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-04-14T11:17:07,480 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-04-14T11:17:07,480 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-04-14T11:17:07,480 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-04-14T11:17:07,480 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-04-14T11:17:07,480 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-04-14T11:17:07,480 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-04-14T11:17:07,480 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-04-14T11:17:07,480 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(402): scanned 1985 2024-04-14T11:17:07,480 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(403): verified 5952 rows 2024-04-14T11:17:07,480 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(402): scanned 1978 2024-04-14T11:17:07,480 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(403): verified 5925 rows 2024-04-14T11:17:07,480 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(402): scanned 1976 2024-04-14T11:17:07,480 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(403): verified 5925 rows 2024-04-14T11:17:07,480 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(402): scanned 1961 2024-04-14T11:17:07,480 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(403): verified 5883 rows 2024-04-14T11:17:07,480 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(402): scanned 1978 2024-04-14T11:17:07,480 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(403): verified 5934 rows 2024-04-14T11:17:07,480 INFO [Listener at localhost/34025 {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-04-14T11:17:07,480 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x73e2e28d to 127.0.0.1:60913 2024-04-14T11:17:07,480 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:07,482 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$18(968): Started disable of TestAcidGuarantees 2024-04-14T11:17:07,483 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$13(2744): Client=jenkins//172.31.2.20 disable TestAcidGuarantees 2024-04-14T11:17:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=127, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-04-14T11:17:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=127 2024-04-14T11:17:07,489 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093427489"}]},"ts":"1713093427489"} 2024-04-14T11:17:07,490 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-04-14T11:17:07,493 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set TestAcidGuarantees to state=DISABLING 2024-04-14T11:17:07,494 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=4974fb0fb1f1e4ab2efd659fb8211d67, UNASSIGN}] 2024-04-14T11:17:07,495 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(729): Took xlock for pid=128, ppid=127, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=4974fb0fb1f1e4ab2efd659fb8211d67, UNASSIGN 2024-04-14T11:17:07,495 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=128 updating hbase:meta row=4974fb0fb1f1e4ab2efd659fb8211d67, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:07,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE; CloseRegionProcedure 4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:17:07,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=127 2024-04-14T11:17:07,648 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:07,648 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] handler.UnassignRegionHandler(119): Close 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:07,648 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1676): Closing 4974fb0fb1f1e4ab2efd659fb8211d67, disabling compactions & flushes 2024-04-14T11:17:07,648 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1698): Closing region TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:07,648 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1719): Waiting without time limit for close lock on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:07,648 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1786): Acquired close lock on TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. after waiting 0 ms 2024-04-14T11:17:07,648 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1796): Updates disabled for region TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:07,649 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(2832): Flushing 4974fb0fb1f1e4ab2efd659fb8211d67 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-04-14T11:17:07,653 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414940987fd5aa34c7f90b51083e6891742_4974fb0fb1f1e4ab2efd659fb8211d67 is 50, key is test_row_1/A:col10/1713093426330/Put/seqid=0 2024-04-14T11:17:07,662 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414940987fd5aa34c7f90b51083e6891742_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414940987fd5aa34c7f90b51083e6891742_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:07,663 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/49d7057fd7df4654b1a75845e7729e16, store: [table=TestAcidGuarantees family=A region=4974fb0fb1f1e4ab2efd659fb8211d67] 2024-04-14T11:17:07,663 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/49d7057fd7df4654b1a75845e7729e16 is 175, key is test_row_1/A:col10/1713093426330/Put/seqid=0 2024-04-14T11:17:07,667 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=451, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/49d7057fd7df4654b1a75845e7729e16 2024-04-14T11:17:07,674 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/340f8d558af24a9ba738102f15590a48 is 50, key is test_row_1/B:col10/1713093426330/Put/seqid=0 2024-04-14T11:17:07,678 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/340f8d558af24a9ba738102f15590a48 2024-04-14T11:17:07,685 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/4a12ef0ca5fb413ab57012d127cb3815 is 50, key is test_row_1/C:col10/1713093426330/Put/seqid=0 2024-04-14T11:17:07,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=127 2024-04-14T11:17:08,091 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/4a12ef0ca5fb413ab57012d127cb3815 2024-04-14T11:17:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=127 2024-04-14T11:17:08,096 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/A/49d7057fd7df4654b1a75845e7729e16 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/49d7057fd7df4654b1a75845e7729e16 2024-04-14T11:17:08,101 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/49d7057fd7df4654b1a75845e7729e16, entries=100, sequenceid=451, filesize=22.0 K 2024-04-14T11:17:08,102 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/B/340f8d558af24a9ba738102f15590a48 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/340f8d558af24a9ba738102f15590a48 2024-04-14T11:17:08,106 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/340f8d558af24a9ba738102f15590a48, entries=100, sequenceid=451, filesize=9.6 K 2024-04-14T11:17:08,107 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/.tmp/C/4a12ef0ca5fb413ab57012d127cb3815 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/4a12ef0ca5fb413ab57012d127cb3815 2024-04-14T11:17:08,112 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/4a12ef0ca5fb413ab57012d127cb3815, entries=100, sequenceid=451, filesize=9.6 K 2024-04-14T11:17:08,112 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(3035): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 4974fb0fb1f1e4ab2efd659fb8211d67 in 464ms, sequenceid=451, compaction requested=true 2024-04-14T11:17:08,113 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/ccd0516a05904f8e80ee3a13f3c5bd39, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/cd7c22d6430b4d989b1d91b7ad92fc09, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/9b2d9b9671394ef8afea7d3fb7fff09c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/c0d3f2970e6a41039be8094606d49057, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/58cc7b53e76d4b67ae104dd337a0943d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/85de372ecdcc425b91d562927d864384, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/c3a4b879ca4b4fa981de030d1b1cdaa0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/fd93c11110724cd6ad238732a4fd78f6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/0e50f6849a954b6898efc68fc8971645, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/cb3fb859030a442fab208d3475a6e4da, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/92c01c982d2f46ac8dc93c85b07ceb2c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/d36ea6e369294aec8b03c399006a45be, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/61c084b28c1f4c69926a8379aa3d8bf0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/d77b15226d144121b67336ebe6c92fed, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/2d7ce83491ac4d7094ccf8bf016e48aa, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/6c6934a93dba4484a4531b88d434cc92, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/430d38c68b5741f69db957822d932170, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/9fcf75e127ab470bad9bce55de3701e1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/fd5ae07d9997445186f072956ac6e497, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/1704bf4b5b1c472db45f39dd60913d02, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/46a2cef538f1479b8cd2f9725fbf2b47, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/8c5b7edc95a64309ae3c4e3f4d68a78b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/f77351c32ad44f3ebcb9fe65b2b037fa, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/436b0fbb92a34cf0bc969f64e2901b5e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/66b11691e39843efb0ae5fa415ed2390, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/52bc7fc385f844c0a85ecafd47189e6b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/52942831a4fa4f02840df6ec324c5332, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/19d6335104754218a24fe7dd027aa9a0] to archive 2024-04-14T11:17:08,114 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:17:08,116 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/ccd0516a05904f8e80ee3a13f3c5bd39 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/ccd0516a05904f8e80ee3a13f3c5bd39 2024-04-14T11:17:08,117 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/cd7c22d6430b4d989b1d91b7ad92fc09 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/cd7c22d6430b4d989b1d91b7ad92fc09 2024-04-14T11:17:08,118 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/9b2d9b9671394ef8afea7d3fb7fff09c to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/9b2d9b9671394ef8afea7d3fb7fff09c 2024-04-14T11:17:08,120 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/c0d3f2970e6a41039be8094606d49057 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/c0d3f2970e6a41039be8094606d49057 2024-04-14T11:17:08,121 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/58cc7b53e76d4b67ae104dd337a0943d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/58cc7b53e76d4b67ae104dd337a0943d 2024-04-14T11:17:08,122 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/85de372ecdcc425b91d562927d864384 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/85de372ecdcc425b91d562927d864384 2024-04-14T11:17:08,124 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/c3a4b879ca4b4fa981de030d1b1cdaa0 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/c3a4b879ca4b4fa981de030d1b1cdaa0 2024-04-14T11:17:08,125 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/fd93c11110724cd6ad238732a4fd78f6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/fd93c11110724cd6ad238732a4fd78f6 2024-04-14T11:17:08,126 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/0e50f6849a954b6898efc68fc8971645 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/0e50f6849a954b6898efc68fc8971645 2024-04-14T11:17:08,127 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/cb3fb859030a442fab208d3475a6e4da to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/cb3fb859030a442fab208d3475a6e4da 2024-04-14T11:17:08,129 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/92c01c982d2f46ac8dc93c85b07ceb2c to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/92c01c982d2f46ac8dc93c85b07ceb2c 2024-04-14T11:17:08,130 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/d36ea6e369294aec8b03c399006a45be to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/d36ea6e369294aec8b03c399006a45be 2024-04-14T11:17:08,131 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/61c084b28c1f4c69926a8379aa3d8bf0 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/61c084b28c1f4c69926a8379aa3d8bf0 2024-04-14T11:17:08,133 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/d77b15226d144121b67336ebe6c92fed to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/d77b15226d144121b67336ebe6c92fed 2024-04-14T11:17:08,134 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/2d7ce83491ac4d7094ccf8bf016e48aa to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/2d7ce83491ac4d7094ccf8bf016e48aa 2024-04-14T11:17:08,135 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/6c6934a93dba4484a4531b88d434cc92 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/6c6934a93dba4484a4531b88d434cc92 2024-04-14T11:17:08,136 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/430d38c68b5741f69db957822d932170 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/430d38c68b5741f69db957822d932170 2024-04-14T11:17:08,137 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/9fcf75e127ab470bad9bce55de3701e1 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/9fcf75e127ab470bad9bce55de3701e1 2024-04-14T11:17:08,139 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/fd5ae07d9997445186f072956ac6e497 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/fd5ae07d9997445186f072956ac6e497 2024-04-14T11:17:08,140 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/1704bf4b5b1c472db45f39dd60913d02 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/1704bf4b5b1c472db45f39dd60913d02 2024-04-14T11:17:08,141 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/46a2cef538f1479b8cd2f9725fbf2b47 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/46a2cef538f1479b8cd2f9725fbf2b47 2024-04-14T11:17:08,142 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/8c5b7edc95a64309ae3c4e3f4d68a78b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/8c5b7edc95a64309ae3c4e3f4d68a78b 2024-04-14T11:17:08,143 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/f77351c32ad44f3ebcb9fe65b2b037fa to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/f77351c32ad44f3ebcb9fe65b2b037fa 2024-04-14T11:17:08,144 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/436b0fbb92a34cf0bc969f64e2901b5e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/436b0fbb92a34cf0bc969f64e2901b5e 2024-04-14T11:17:08,146 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/66b11691e39843efb0ae5fa415ed2390 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/66b11691e39843efb0ae5fa415ed2390 2024-04-14T11:17:08,147 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/52bc7fc385f844c0a85ecafd47189e6b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/52bc7fc385f844c0a85ecafd47189e6b 2024-04-14T11:17:08,148 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/52942831a4fa4f02840df6ec324c5332 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/52942831a4fa4f02840df6ec324c5332 2024-04-14T11:17:08,149 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/19d6335104754218a24fe7dd027aa9a0 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/19d6335104754218a24fe7dd027aa9a0 2024-04-14T11:17:08,150 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/932d6695b4ba49adbbd19ff87491e1be, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/1e1272da20b241cb83cbaba6606b9150, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/b22f8445f6ad46b09578386e6884a908, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/f2c2a63e31ae437ba10549842686e603, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/a83201c2ecf6477ab8135b85c08d7964, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/34f9b5d15919402cb1dfda51ca6d8c1e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/abc746eee3854b2ea83f7709061fc279, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/78d94f55bd384a7da5daac033166ae72, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/737fa11041704b33bd2ebf4582eb854a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/215fc5e9838f4dd6b580db0b86c117db, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/6058f622d87340aeac8031141529c13d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/f765bf9640004041a63cee893b3926a6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/f0570734c014463f8fb0b6778967f304, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/3ba534c3dfad4dc5b8907b35ec0a105d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/570ef17fa5f9477aab480cbe07afc28e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/84edc7ab1ff641c398070c87f0f6f0ac, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/fb2d6a32b6f544f1b70d966e94b75cfe, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/88a8c5a3ae294de1b7d0cb0ac7d89103, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/b6bca641e33048b1bce1d9a617c5daa2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/179598e01571418ea8c8cde5a49d840e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/4f4b4fc41b9847c29f16912795024040, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/9c6a58f3a9d6481ea8223128febf8096, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/16937dce0804415bb1f8a8d8390b9847, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/e17fa746da5b49c0895723e712ebf65a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/9520682c7081496a87ad60f29412e597, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/c31070ea6cab4e46a14b58c5139ea96e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/8bb35e5254274609a5f9cba4f4d41539, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/83513c623ea54416858c1d25f51682ce] to archive 2024-04-14T11:17:08,151 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:17:08,152 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/932d6695b4ba49adbbd19ff87491e1be to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/932d6695b4ba49adbbd19ff87491e1be 2024-04-14T11:17:08,153 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/1e1272da20b241cb83cbaba6606b9150 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/1e1272da20b241cb83cbaba6606b9150 2024-04-14T11:17:08,154 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/b22f8445f6ad46b09578386e6884a908 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/b22f8445f6ad46b09578386e6884a908 2024-04-14T11:17:08,155 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/f2c2a63e31ae437ba10549842686e603 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/f2c2a63e31ae437ba10549842686e603 2024-04-14T11:17:08,156 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/a83201c2ecf6477ab8135b85c08d7964 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/a83201c2ecf6477ab8135b85c08d7964 2024-04-14T11:17:08,157 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/34f9b5d15919402cb1dfda51ca6d8c1e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/34f9b5d15919402cb1dfda51ca6d8c1e 2024-04-14T11:17:08,158 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/abc746eee3854b2ea83f7709061fc279 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/abc746eee3854b2ea83f7709061fc279 2024-04-14T11:17:08,160 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/78d94f55bd384a7da5daac033166ae72 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/78d94f55bd384a7da5daac033166ae72 2024-04-14T11:17:08,161 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/737fa11041704b33bd2ebf4582eb854a to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/737fa11041704b33bd2ebf4582eb854a 2024-04-14T11:17:08,162 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/215fc5e9838f4dd6b580db0b86c117db to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/215fc5e9838f4dd6b580db0b86c117db 2024-04-14T11:17:08,163 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/6058f622d87340aeac8031141529c13d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/6058f622d87340aeac8031141529c13d 2024-04-14T11:17:08,164 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/f765bf9640004041a63cee893b3926a6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/f765bf9640004041a63cee893b3926a6 2024-04-14T11:17:08,165 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/f0570734c014463f8fb0b6778967f304 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/f0570734c014463f8fb0b6778967f304 2024-04-14T11:17:08,166 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/3ba534c3dfad4dc5b8907b35ec0a105d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/3ba534c3dfad4dc5b8907b35ec0a105d 2024-04-14T11:17:08,167 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/570ef17fa5f9477aab480cbe07afc28e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/570ef17fa5f9477aab480cbe07afc28e 2024-04-14T11:17:08,168 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/84edc7ab1ff641c398070c87f0f6f0ac to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/84edc7ab1ff641c398070c87f0f6f0ac 2024-04-14T11:17:08,169 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/fb2d6a32b6f544f1b70d966e94b75cfe to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/fb2d6a32b6f544f1b70d966e94b75cfe 2024-04-14T11:17:08,170 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/88a8c5a3ae294de1b7d0cb0ac7d89103 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/88a8c5a3ae294de1b7d0cb0ac7d89103 2024-04-14T11:17:08,171 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/b6bca641e33048b1bce1d9a617c5daa2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/b6bca641e33048b1bce1d9a617c5daa2 2024-04-14T11:17:08,172 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/179598e01571418ea8c8cde5a49d840e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/179598e01571418ea8c8cde5a49d840e 2024-04-14T11:17:08,174 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/4f4b4fc41b9847c29f16912795024040 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/4f4b4fc41b9847c29f16912795024040 2024-04-14T11:17:08,175 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/9c6a58f3a9d6481ea8223128febf8096 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/9c6a58f3a9d6481ea8223128febf8096 2024-04-14T11:17:08,176 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/16937dce0804415bb1f8a8d8390b9847 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/16937dce0804415bb1f8a8d8390b9847 2024-04-14T11:17:08,177 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/e17fa746da5b49c0895723e712ebf65a to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/e17fa746da5b49c0895723e712ebf65a 2024-04-14T11:17:08,178 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/9520682c7081496a87ad60f29412e597 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/9520682c7081496a87ad60f29412e597 2024-04-14T11:17:08,179 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/c31070ea6cab4e46a14b58c5139ea96e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/c31070ea6cab4e46a14b58c5139ea96e 2024-04-14T11:17:08,180 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/8bb35e5254274609a5f9cba4f4d41539 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/8bb35e5254274609a5f9cba4f4d41539 2024-04-14T11:17:08,181 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/83513c623ea54416858c1d25f51682ce to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/83513c623ea54416858c1d25f51682ce 2024-04-14T11:17:08,183 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/34e54770ce504ee894b2c8ccf6369c26, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/d8604f42fe2a4ffe8b1d3c8309fb4163, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/184631ea03e94ecd82fd16cb68608369, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/15279c1426da4138b481cd74ca45a7af, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/20b7212b01904353a744a26742ad93af, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/4ed34d85a6c84447929c66c2490a86e2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/dfe2e981a48f48f29810fdb392cc03ec, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/257822f3e263462596d394a568df9265, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/268c02e972b94c90aaf2027cce0c1036, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/71b7370292e442e9a97749c984a2c2bf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/dcfed90b4af946ecbffe26f486578313, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/c6ceef3c544d4139818f38d541215ff8, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/0ab35fc5d0c04300b520b3f7db079adf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/d2edc037e88b4d4caba2c5c9bc767005, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/3b380b5ad1784bae849f18dcc1ccd5c2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/4f8c8e75c8c8456a9e0f38a09a0c21a3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/0484d10019394b24978894b5a80e389e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/95085c15d2514178a6114c1c687f23a2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/2773ad32711548df89ddd5e7fa3f7527, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/62e088ae27ec4b68ae3312e60f5e3c3e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/d5d07d7c8e6948a1ab0ba0953f7d85b9, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/68f5764ca31e4ce98275c3cbe82aa7d9, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/0efe8e87915742d287db6229e2056d82, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/a54982e262c54044b4b363947b37a042, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/3f1f955f9efc4a058d6c9c3a3e471cde, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/b5f302695c9147e2adcfedda3b1bfb2f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/a1c7386c88a04248b32c9addf6e19028, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/096c0ce2bbf54e11b6fa7f90d15c8150] to archive 2024-04-14T11:17:08,183 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:17:08,185 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/34e54770ce504ee894b2c8ccf6369c26 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/34e54770ce504ee894b2c8ccf6369c26 2024-04-14T11:17:08,186 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/d8604f42fe2a4ffe8b1d3c8309fb4163 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/d8604f42fe2a4ffe8b1d3c8309fb4163 2024-04-14T11:17:08,187 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/184631ea03e94ecd82fd16cb68608369 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/184631ea03e94ecd82fd16cb68608369 2024-04-14T11:17:08,188 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/15279c1426da4138b481cd74ca45a7af to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/15279c1426da4138b481cd74ca45a7af 2024-04-14T11:17:08,190 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/20b7212b01904353a744a26742ad93af to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/20b7212b01904353a744a26742ad93af 2024-04-14T11:17:08,191 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/4ed34d85a6c84447929c66c2490a86e2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/4ed34d85a6c84447929c66c2490a86e2 2024-04-14T11:17:08,192 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/dfe2e981a48f48f29810fdb392cc03ec to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/dfe2e981a48f48f29810fdb392cc03ec 2024-04-14T11:17:08,193 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/257822f3e263462596d394a568df9265 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/257822f3e263462596d394a568df9265 2024-04-14T11:17:08,194 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/268c02e972b94c90aaf2027cce0c1036 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/268c02e972b94c90aaf2027cce0c1036 2024-04-14T11:17:08,195 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/71b7370292e442e9a97749c984a2c2bf to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/71b7370292e442e9a97749c984a2c2bf 2024-04-14T11:17:08,196 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/dcfed90b4af946ecbffe26f486578313 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/dcfed90b4af946ecbffe26f486578313 2024-04-14T11:17:08,197 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/c6ceef3c544d4139818f38d541215ff8 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/c6ceef3c544d4139818f38d541215ff8 2024-04-14T11:17:08,198 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/0ab35fc5d0c04300b520b3f7db079adf to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/0ab35fc5d0c04300b520b3f7db079adf 2024-04-14T11:17:08,199 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/d2edc037e88b4d4caba2c5c9bc767005 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/d2edc037e88b4d4caba2c5c9bc767005 2024-04-14T11:17:08,200 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/3b380b5ad1784bae849f18dcc1ccd5c2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/3b380b5ad1784bae849f18dcc1ccd5c2 2024-04-14T11:17:08,202 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/4f8c8e75c8c8456a9e0f38a09a0c21a3 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/4f8c8e75c8c8456a9e0f38a09a0c21a3 2024-04-14T11:17:08,203 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/0484d10019394b24978894b5a80e389e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/0484d10019394b24978894b5a80e389e 2024-04-14T11:17:08,204 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/95085c15d2514178a6114c1c687f23a2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/95085c15d2514178a6114c1c687f23a2 2024-04-14T11:17:08,205 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/2773ad32711548df89ddd5e7fa3f7527 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/2773ad32711548df89ddd5e7fa3f7527 2024-04-14T11:17:08,206 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/62e088ae27ec4b68ae3312e60f5e3c3e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/62e088ae27ec4b68ae3312e60f5e3c3e 2024-04-14T11:17:08,207 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/d5d07d7c8e6948a1ab0ba0953f7d85b9 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/d5d07d7c8e6948a1ab0ba0953f7d85b9 2024-04-14T11:17:08,208 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/68f5764ca31e4ce98275c3cbe82aa7d9 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/68f5764ca31e4ce98275c3cbe82aa7d9 2024-04-14T11:17:08,210 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/0efe8e87915742d287db6229e2056d82 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/0efe8e87915742d287db6229e2056d82 2024-04-14T11:17:08,211 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/a54982e262c54044b4b363947b37a042 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/a54982e262c54044b4b363947b37a042 2024-04-14T11:17:08,212 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/3f1f955f9efc4a058d6c9c3a3e471cde to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/3f1f955f9efc4a058d6c9c3a3e471cde 2024-04-14T11:17:08,213 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/b5f302695c9147e2adcfedda3b1bfb2f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/b5f302695c9147e2adcfedda3b1bfb2f 2024-04-14T11:17:08,214 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/a1c7386c88a04248b32c9addf6e19028 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/a1c7386c88a04248b32c9addf6e19028 2024-04-14T11:17:08,216 DEBUG [StoreCloser-TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/096c0ce2bbf54e11b6fa7f90d15c8150 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/096c0ce2bbf54e11b6fa7f90d15c8150 2024-04-14T11:17:08,220 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/recovered.edits/454.seqid, newMaxSeqId=454, maxSeqId=4 2024-04-14T11:17:08,221 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1917): Closed TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67. 2024-04-14T11:17:08,221 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1630): Region close journal for 4974fb0fb1f1e4ab2efd659fb8211d67: 2024-04-14T11:17:08,222 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] handler.UnassignRegionHandler(163): Closed 4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,223 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=128 updating hbase:meta row=4974fb0fb1f1e4ab2efd659fb8211d67, regionState=CLOSED 2024-04-14T11:17:08,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=129, resume processing ppid=128 2024-04-14T11:17:08,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=129, ppid=128, state=SUCCESS; CloseRegionProcedure 4974fb0fb1f1e4ab2efd659fb8211d67, server=jenkins-hbase6.apache.org,33239,1713093321027 in 728 msec 2024-04-14T11:17:08,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=128, resume processing ppid=127 2024-04-14T11:17:08,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=128, ppid=127, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=4974fb0fb1f1e4ab2efd659fb8211d67, UNASSIGN in 731 msec 2024-04-14T11:17:08,227 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093428227"}]},"ts":"1713093428227"} 2024-04-14T11:17:08,228 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-04-14T11:17:08,230 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set TestAcidGuarantees to state=DISABLED 2024-04-14T11:17:08,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=127, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 747 msec 2024-04-14T11:17:08,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=127 2024-04-14T11:17:08,594 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$TableFuture(3746): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 127 completed 2024-04-14T11:17:08,594 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$5(2494): Client=jenkins//172.31.2.20 delete TestAcidGuarantees 2024-04-14T11:17:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=130, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:17:08,597 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=130, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:17:08,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=130 2024-04-14T11:17:08,598 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=130, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:17:08,601 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,603 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A, FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B, FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C, FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/recovered.edits] 2024-04-14T11:17:08,608 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/49d7057fd7df4654b1a75845e7729e16 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/49d7057fd7df4654b1a75845e7729e16 2024-04-14T11:17:08,610 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/7a299da7cbcc4b2bb6a012c7037f8864 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/7a299da7cbcc4b2bb6a012c7037f8864 2024-04-14T11:17:08,611 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/d50e706111d843c89aaf5a5495be950a to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/A/d50e706111d843c89aaf5a5495be950a 2024-04-14T11:17:08,614 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/1fc5b3436de5403f93558e7f77ded686 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/1fc5b3436de5403f93558e7f77ded686 2024-04-14T11:17:08,616 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/340f8d558af24a9ba738102f15590a48 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/340f8d558af24a9ba738102f15590a48 2024-04-14T11:17:08,618 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/5900360e543b418d9524b7e0590255a8 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/B/5900360e543b418d9524b7e0590255a8 2024-04-14T11:17:08,620 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/4a12ef0ca5fb413ab57012d127cb3815 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/4a12ef0ca5fb413ab57012d127cb3815 2024-04-14T11:17:08,622 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/8dde4394c8cd4bbfb94982a18d2bcbe2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/8dde4394c8cd4bbfb94982a18d2bcbe2 2024-04-14T11:17:08,623 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/9f20ec157ab74be7b71cf6f51edad771 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/C/9f20ec157ab74be7b71cf6f51edad771 2024-04-14T11:17:08,626 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/recovered.edits/454.seqid to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67/recovered.edits/454.seqid 2024-04-14T11:17:08,627 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,627 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(293): Archived TestAcidGuarantees regions 2024-04-14T11:17:08,627 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-04-14T11:17:08,628 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-04-14T11:17:08,632 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041413ff8e04d91b4eaa9494698a46987608_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041413ff8e04d91b4eaa9494698a46987608_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,633 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404142155929a986242a18696307013ab7a22_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404142155929a986242a18696307013ab7a22_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,634 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041422ace97b1e894ba1ab684b3a24cf14b0_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041422ace97b1e894ba1ab684b3a24cf14b0_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,636 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404142c1a0eaeb86049119ebcd1b0d8074977_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404142c1a0eaeb86049119ebcd1b0d8074977_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,637 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041448b7c596087e4eeeb768a1e68f5c187a_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041448b7c596087e4eeeb768a1e68f5c187a_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,638 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404144fc7eae2a41b440ba014c5e28cc9b740_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404144fc7eae2a41b440ba014c5e28cc9b740_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,639 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404145261ec0e4f5f418c97bd269474c2b2ad_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404145261ec0e4f5f418c97bd269474c2b2ad_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,641 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404145cfcff8bb3a4490d95609905d897dcad_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404145cfcff8bb3a4490d95609905d897dcad_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,642 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414733b6d7a3a2943d385fde6a97e2ff846_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414733b6d7a3a2943d385fde6a97e2ff846_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,643 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404147eb393b4e14146119e22de001c0171a5_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404147eb393b4e14146119e22de001c0171a5_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,645 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404147f1d633f8af64034bccfd43244d87af2_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404147f1d633f8af64034bccfd43244d87af2_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,646 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414940987fd5aa34c7f90b51083e6891742_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414940987fd5aa34c7f90b51083e6891742_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,647 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414943e5d1f06224f8f82258783e9f33b48_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414943e5d1f06224f8f82258783e9f33b48_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,649 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041496ee990144724baea4513091fe7f977e_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041496ee990144724baea4513091fe7f977e_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,650 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414988952a38fd6418f8817bed9872eae23_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414988952a38fd6418f8817bed9872eae23_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,651 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414ad6e0977c3494ce4bcb0b75b2ac68976_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414ad6e0977c3494ce4bcb0b75b2ac68976_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,652 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414b0aaf40592b2479db06740561574c6d4_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414b0aaf40592b2479db06740561574c6d4_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,654 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414b4a3911124b84bf8b86428cd8d987974_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414b4a3911124b84bf8b86428cd8d987974_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,655 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414bb1dbe7016784ec3b2802ec7c8d495bd_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414bb1dbe7016784ec3b2802ec7c8d495bd_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,656 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414c0d58900695c4961b22ab8df9c19ec6a_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414c0d58900695c4961b22ab8df9c19ec6a_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,658 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414d0c170076e5641b1bc8e2e47712b1133_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414d0c170076e5641b1bc8e2e47712b1133_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,659 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414d5b3b925996840f9a57f1fb7b6c28c41_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414d5b3b925996840f9a57f1fb7b6c28c41_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,660 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414e78a4712882d4f8bbf36394ad46bec64_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414e78a4712882d4f8bbf36394ad46bec64_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,662 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414f84756467730434a8aefd3c842cd9c17_4974fb0fb1f1e4ab2efd659fb8211d67 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414f84756467730434a8aefd3c842cd9c17_4974fb0fb1f1e4ab2efd659fb8211d67 2024-04-14T11:17:08,662 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-04-14T11:17:08,666 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=130, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:17:08,667 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-04-14T11:17:08,671 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(387): Removing 'TestAcidGuarantees' descriptor. 2024-04-14T11:17:08,672 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=130, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:17:08,673 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(377): Removing 'TestAcidGuarantees' from region states. 2024-04-14T11:17:08,673 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1713093428673"}]},"ts":"9223372036854775807"} 2024-04-14T11:17:08,683 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-04-14T11:17:08,683 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 4974fb0fb1f1e4ab2efd659fb8211d67, NAME => 'TestAcidGuarantees,,1713093404162.4974fb0fb1f1e4ab2efd659fb8211d67.', STARTKEY => '', ENDKEY => ''}] 2024-04-14T11:17:08,684 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(381): Marking 'TestAcidGuarantees' as deleted. 2024-04-14T11:17:08,684 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1713093428684"}]},"ts":"9223372036854775807"} 2024-04-14T11:17:08,687 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-04-14T11:17:08,689 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(127): Finished pid=130, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:17:08,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=130, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 95 msec 2024-04-14T11:17:08,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=130 2024-04-14T11:17:08,699 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$TableFuture(3746): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 130 completed 2024-04-14T11:17:08,713 INFO [Listener at localhost/34025 {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=223 (was 223), OpenFileDescriptor=530 (was 527) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=588 (was 536) - SystemLoadAverage LEAK? -, ProcessCount=168 (was 168), AvailableMemoryMB=1561 (was 2088) 2024-04-14T11:17:08,727 INFO [Listener at localhost/34025 {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=223, OpenFileDescriptor=530, MaxFileDescriptor=60000, SystemLoadAverage=588, ProcessCount=168, AvailableMemoryMB=1561 2024-04-14T11:17:08,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-04-14T11:17:08,730 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$4(2378): Client=jenkins//172.31.2.20 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-04-14T11:17:08,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=131, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-04-14T11:17:08,733 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(84): pid=131, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-04-14T11:17:08,733 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:17:08,733 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 131 2024-04-14T11:17:08,734 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(84): pid=131, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-04-14T11:17:08,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=131 2024-04-14T11:17:08,742 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7088): creating {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a 2024-04-14T11:17:08,749 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(891): Instantiated TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:17:08,749 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Closing 893136536711aed76a1c4f901f8c1645, disabling compactions & flushes 2024-04-14T11:17:08,749 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1698): Closing region TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:08,749 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1719): Waiting without time limit for close lock on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:08,749 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1786): Acquired close lock on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. after waiting 0 ms 2024-04-14T11:17:08,749 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1796): Updates disabled for region TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:08,749 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1917): Closed TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:08,749 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1630): Region close journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:08,750 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(84): pid=131, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-04-14T11:17:08,750 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1713093428750"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1713093428750"}]},"ts":"1713093428750"} 2024-04-14T11:17:08,752 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-04-14T11:17:08,752 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(84): pid=131, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-04-14T11:17:08,753 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093428752"}]},"ts":"1713093428752"} 2024-04-14T11:17:08,754 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-04-14T11:17:08,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=893136536711aed76a1c4f901f8c1645, ASSIGN}] 2024-04-14T11:17:08,761 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(729): Took xlock for pid=132, ppid=131, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=893136536711aed76a1c4f901f8c1645, ASSIGN 2024-04-14T11:17:08,762 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=132, ppid=131, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=893136536711aed76a1c4f901f8c1645, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,33239,1713093321027; forceNewPlan=false, retain=false 2024-04-14T11:17:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=131 2024-04-14T11:17:08,914 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=132 updating hbase:meta row=893136536711aed76a1c4f901f8c1645, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:08,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE; OpenRegionProcedure 893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:17:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=131 2024-04-14T11:17:09,067 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,070 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:09,070 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(7267): Opening region: {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} 2024-04-14T11:17:09,071 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:09,071 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(891): Instantiated TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:17:09,071 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(7309): checking encryption for 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:09,071 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(7312): checking classloading for 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:09,072 INFO [StoreOpener-893136536711aed76a1c4f901f8c1645-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:09,074 INFO [StoreOpener-893136536711aed76a1c4f901f8c1645-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 893136536711aed76a1c4f901f8c1645 columnFamilyName A 2024-04-14T11:17:09,074 DEBUG [StoreOpener-893136536711aed76a1c4f901f8c1645-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:17:09,075 INFO [StoreOpener-893136536711aed76a1c4f901f8c1645-1 {}] regionserver.HStore(324): Store=893136536711aed76a1c4f901f8c1645/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:17:09,075 INFO [StoreOpener-893136536711aed76a1c4f901f8c1645-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:09,076 INFO [StoreOpener-893136536711aed76a1c4f901f8c1645-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 893136536711aed76a1c4f901f8c1645 columnFamilyName B 2024-04-14T11:17:09,076 DEBUG [StoreOpener-893136536711aed76a1c4f901f8c1645-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:17:09,077 INFO [StoreOpener-893136536711aed76a1c4f901f8c1645-1 {}] regionserver.HStore(324): Store=893136536711aed76a1c4f901f8c1645/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:17:09,077 INFO [StoreOpener-893136536711aed76a1c4f901f8c1645-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:09,079 INFO [StoreOpener-893136536711aed76a1c4f901f8c1645-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 893136536711aed76a1c4f901f8c1645 columnFamilyName C 2024-04-14T11:17:09,079 DEBUG [StoreOpener-893136536711aed76a1c4f901f8c1645-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:17:09,080 INFO [StoreOpener-893136536711aed76a1c4f901f8c1645-1 {}] regionserver.HStore(324): Store=893136536711aed76a1c4f901f8c1645/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:17:09,080 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:09,081 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:09,083 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-04-14T11:17:09,084 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(1082): writing seq id for 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:09,086 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-04-14T11:17:09,087 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(1099): Opened 893136536711aed76a1c4f901f8c1645; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69760448, jitterRate=0.039511680603027344}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-04-14T11:17:09,087 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegion(998): Region open journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:09,088 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegionServer(2593): Post open deploy tasks for TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., pid=133, masterSystemTime=1713093429066 2024-04-14T11:17:09,090 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] regionserver.HRegionServer(2620): Finished post open deploy task for TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:09,090 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=133}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:09,090 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=132 updating hbase:meta row=893136536711aed76a1c4f901f8c1645, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=133, resume processing ppid=132 2024-04-14T11:17:09,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=133, ppid=132, state=SUCCESS; OpenRegionProcedure 893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 in 176 msec 2024-04-14T11:17:09,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=132, resume processing ppid=131 2024-04-14T11:17:09,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=132, ppid=131, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=893136536711aed76a1c4f901f8c1645, ASSIGN in 333 msec 2024-04-14T11:17:09,095 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(84): pid=131, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-04-14T11:17:09,096 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093429095"}]},"ts":"1713093429095"} 2024-04-14T11:17:09,097 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-04-14T11:17:09,100 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(84): pid=131, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-04-14T11:17:09,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=131, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 370 msec 2024-04-14T11:17:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=131 2024-04-14T11:17:09,338 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 131 completed 2024-04-14T11:17:09,339 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x63a6b404 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:09,345 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349094ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:09,346 DEBUG [Listener at localhost/34025 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:09,348 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:49526, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:09,349 DEBUG [Listener at localhost/34025 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-04-14T11:17:09,350 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:60588, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-04-14T11:17:09,352 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x523efeac to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:09,357 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46e767b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:09,357 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7fb749ac to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:09,363 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a591368, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:09,364 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x444c7fb3 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:09,368 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@173b4bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:09,368 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2539c0e3 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:09,372 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22f8210, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:09,373 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x14e170b5 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:09,376 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fb49412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:09,377 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x289884d7 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:09,382 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@735781d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:09,383 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7106cfe7 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:09,387 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dd4432c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:09,388 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x47b46ade to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:09,392 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8afc38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:09,392 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7d3aa52c to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:09,397 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bcc2cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:09,398 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x31165428 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:09,403 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@192d94e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:09,408 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees 2024-04-14T11:17:09,409 DEBUG [hconnection-0x652fd344-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:09,410 DEBUG [hconnection-0x77f214b2-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:09,410 DEBUG [hconnection-0xfcd080b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:09,411 DEBUG [hconnection-0x49704251-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:09,412 DEBUG [hconnection-0x5a4b8641-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:09,415 DEBUG [hconnection-0x48a01cf0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:09,415 DEBUG [hconnection-0x7e95525-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:09,415 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:09,415 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:49530, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:09,415 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:49528, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:09,415 DEBUG [hconnection-0x45cef480-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:09,415 DEBUG [hconnection-0x7de7b772-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=134 2024-04-14T11:17:09,417 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:49550, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:09,417 DEBUG [hconnection-0x13c7338f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:09,417 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:49536, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:09,419 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:49552, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:09,419 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:49570, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:09,423 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:49572, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:09,423 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=134, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:09,423 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:49590, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:09,423 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:09,425 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:49598, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:09,425 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:49600, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:09,434 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:09,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-04-14T11:17:09,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:09,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093489447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093489448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:09,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093489448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093489449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093489449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/bd58560fe8384777b052c75d10d73bb6 is 50, key is test_row_0/A:col10/1713093429431/Put/seqid=0 2024-04-14T11:17:09,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/bd58560fe8384777b052c75d10d73bb6 2024-04-14T11:17:09,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/939bf8d4a8e347a3a23c7e6e888382a9 is 50, key is test_row_0/B:col10/1713093429431/Put/seqid=0 2024-04-14T11:17:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=134 2024-04-14T11:17:09,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093489550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093489550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093489550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093489551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093489551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,580 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-04-14T11:17:09,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:09,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:09,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:09,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(53): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:09,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=134 2024-04-14T11:17:09,733 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-04-14T11:17:09,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:09,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:09,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:09,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(53): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:09,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:09,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093489754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093489754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093489755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093489755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093489755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/939bf8d4a8e347a3a23c7e6e888382a9 2024-04-14T11:17:09,887 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:09,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-04-14T11:17:09,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:09,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:09,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:09,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] handler.RSProcedureHandler(53): pid=135 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:09,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=135 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:09,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=135 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:09,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/69e4b31a0b2a4e75bddaf53cc5263412 is 50, key is test_row_0/C:col10/1713093429431/Put/seqid=0 2024-04-14T11:17:09,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/69e4b31a0b2a4e75bddaf53cc5263412 2024-04-14T11:17:09,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/bd58560fe8384777b052c75d10d73bb6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/bd58560fe8384777b052c75d10d73bb6 2024-04-14T11:17:09,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/bd58560fe8384777b052c75d10d73bb6, entries=150, sequenceid=15, filesize=11.7 K 2024-04-14T11:17:09,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/939bf8d4a8e347a3a23c7e6e888382a9 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/939bf8d4a8e347a3a23c7e6e888382a9 2024-04-14T11:17:10,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/939bf8d4a8e347a3a23c7e6e888382a9, entries=150, sequenceid=15, filesize=11.7 K 2024-04-14T11:17:10,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/69e4b31a0b2a4e75bddaf53cc5263412 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/69e4b31a0b2a4e75bddaf53cc5263412 2024-04-14T11:17:10,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/69e4b31a0b2a4e75bddaf53cc5263412, entries=150, sequenceid=15, filesize=11.7 K 2024-04-14T11:17:10,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 893136536711aed76a1c4f901f8c1645 in 578ms, sequenceid=15, compaction requested=false 2024-04-14T11:17:10,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=134 2024-04-14T11:17:10,042 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-04-14T11:17:10,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:10,044 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-04-14T11:17:10,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/f15d0e26dd92400b9848c8d1e6a7800b is 50, key is test_row_0/A:col10/1713093429445/Put/seqid=0 2024-04-14T11:17:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:10,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:10,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093490077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093490077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093490078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093490084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093490084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093490186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093490187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093490187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093490192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093490193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093490390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093490392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093490392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093490399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093490399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,490 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/f15d0e26dd92400b9848c8d1e6a7800b 2024-04-14T11:17:10,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/f271f3503bd149b0a1aa0231df6f75c6 is 50, key is test_row_0/B:col10/1713093429445/Put/seqid=0 2024-04-14T11:17:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=134 2024-04-14T11:17:10,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093490695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093490697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093490697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093490702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:10,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093490702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:10,911 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/f271f3503bd149b0a1aa0231df6f75c6 2024-04-14T11:17:10,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/90ee543f1dab4f9a98372dbfe73d1f2a is 50, key is test_row_0/C:col10/1713093429445/Put/seqid=0 2024-04-14T11:17:11,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093491199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:11,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093491200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:11,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:11,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093491201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:11,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093491206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:11,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093491208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:11,349 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/90ee543f1dab4f9a98372dbfe73d1f2a 2024-04-14T11:17:11,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/f15d0e26dd92400b9848c8d1e6a7800b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/f15d0e26dd92400b9848c8d1e6a7800b 2024-04-14T11:17:11,362 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/f15d0e26dd92400b9848c8d1e6a7800b, entries=150, sequenceid=37, filesize=11.7 K 2024-04-14T11:17:11,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/f271f3503bd149b0a1aa0231df6f75c6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/f271f3503bd149b0a1aa0231df6f75c6 2024-04-14T11:17:11,370 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/f271f3503bd149b0a1aa0231df6f75c6, entries=150, sequenceid=37, filesize=11.7 K 2024-04-14T11:17:11,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/90ee543f1dab4f9a98372dbfe73d1f2a as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/90ee543f1dab4f9a98372dbfe73d1f2a 2024-04-14T11:17:11,377 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/90ee543f1dab4f9a98372dbfe73d1f2a, entries=150, sequenceid=37, filesize=11.7 K 2024-04-14T11:17:11,378 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(3035): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 893136536711aed76a1c4f901f8c1645 in 1334ms, sequenceid=37, compaction requested=false 2024-04-14T11:17:11,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:11,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:11,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=135 2024-04-14T11:17:11,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=135 2024-04-14T11:17:11,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=135, resume processing ppid=134 2024-04-14T11:17:11,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=135, ppid=134, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9560 sec 2024-04-14T11:17:11,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=134, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees in 1.9730 sec 2024-04-14T11:17:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=134 2024-04-14T11:17:11,525 INFO [Thread-2284 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 134 completed 2024-04-14T11:17:11,526 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:11,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees 2024-04-14T11:17:11,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=136 2024-04-14T11:17:11,530 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:11,531 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=136, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:11,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=136 2024-04-14T11:17:11,682 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:11,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-04-14T11:17:11,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:11,683 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-04-14T11:17:11,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/abbbe152e181417e87539ecf8ceaa8a2 is 50, key is test_row_0/A:col10/1713093430081/Put/seqid=0 2024-04-14T11:17:11,692 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/abbbe152e181417e87539ecf8ceaa8a2 2024-04-14T11:17:11,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/13c602a016bf4fc7aa7e8f3b609bfee0 is 50, key is test_row_0/B:col10/1713093430081/Put/seqid=0 2024-04-14T11:17:11,704 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/13c602a016bf4fc7aa7e8f3b609bfee0 2024-04-14T11:17:11,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/b7975f3e707c44e1a6667ca2ccd785f5 is 50, key is test_row_0/C:col10/1713093430081/Put/seqid=0 2024-04-14T11:17:11,718 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/b7975f3e707c44e1a6667ca2ccd785f5 2024-04-14T11:17:11,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/abbbe152e181417e87539ecf8ceaa8a2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/abbbe152e181417e87539ecf8ceaa8a2 2024-04-14T11:17:11,728 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/abbbe152e181417e87539ecf8ceaa8a2, entries=150, sequenceid=52, filesize=11.7 K 2024-04-14T11:17:11,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/13c602a016bf4fc7aa7e8f3b609bfee0 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/13c602a016bf4fc7aa7e8f3b609bfee0 2024-04-14T11:17:11,734 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/13c602a016bf4fc7aa7e8f3b609bfee0, entries=150, sequenceid=52, filesize=11.7 K 2024-04-14T11:17:11,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/b7975f3e707c44e1a6667ca2ccd785f5 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/b7975f3e707c44e1a6667ca2ccd785f5 2024-04-14T11:17:11,739 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/b7975f3e707c44e1a6667ca2ccd785f5, entries=150, sequenceid=52, filesize=11.7 K 2024-04-14T11:17:11,740 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(3035): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for 893136536711aed76a1c4f901f8c1645 in 57ms, sequenceid=52, compaction requested=true 2024-04-14T11:17:11,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:11,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:11,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=137 2024-04-14T11:17:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=137 2024-04-14T11:17:11,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=137, resume processing ppid=136 2024-04-14T11:17:11,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=137, ppid=136, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 210 msec 2024-04-14T11:17:11,744 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=136, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees in 217 msec 2024-04-14T11:17:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=136 2024-04-14T11:17:11,832 INFO [Thread-2284 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 136 completed 2024-04-14T11:17:11,834 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees 2024-04-14T11:17:11,838 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=138 2024-04-14T11:17:11,839 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=138, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:11,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=138 2024-04-14T11:17:11,991 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:11,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-04-14T11:17:11,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:11,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:11,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:11,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=139 2024-04-14T11:17:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=139 2024-04-14T11:17:11,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=139, resume processing ppid=138 2024-04-14T11:17:11,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=139, ppid=138, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-04-14T11:17:11,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=138, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees in 161 msec 2024-04-14T11:17:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=138 2024-04-14T11:17:12,141 INFO [Thread-2284 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 138 completed 2024-04-14T11:17:12,143 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-04-14T11:17:12,147 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=140 2024-04-14T11:17:12,147 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:12,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:12,223 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:12,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-04-14T11:17:12,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/d6ce1b581ac141fd8296611e951f3008 is 50, key is test_row_0/A:col10/1713093432222/Put/seqid=0 2024-04-14T11:17:12,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/d6ce1b581ac141fd8296611e951f3008 2024-04-14T11:17:12,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/cc275a271ab0495a8166ea57ded680ce is 50, key is test_row_0/B:col10/1713093432222/Put/seqid=0 2024-04-14T11:17:12,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/cc275a271ab0495a8166ea57ded680ce 2024-04-14T11:17:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=140 2024-04-14T11:17:12,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/737a5ed110644ebfb39749d5a3362aa2 is 50, key is test_row_0/C:col10/1713093432222/Put/seqid=0 2024-04-14T11:17:12,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093492257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093492258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093492267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093492268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093492269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,299 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-04-14T11:17:12,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:12,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:12,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:12,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(53): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:12,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:12,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:12,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093492376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093492376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093492376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093492377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093492377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=140 2024-04-14T11:17:12,452 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-04-14T11:17:12,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:12,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:12,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:12,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(53): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:12,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:12,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093492579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093492580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093492580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093492581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093492581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,605 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-04-14T11:17:12,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:12,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:12,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:12,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(53): pid=141 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:12,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:12,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/737a5ed110644ebfb39749d5a3362aa2 2024-04-14T11:17:12,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/d6ce1b581ac141fd8296611e951f3008 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/d6ce1b581ac141fd8296611e951f3008 2024-04-14T11:17:12,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/d6ce1b581ac141fd8296611e951f3008, entries=200, sequenceid=64, filesize=14.0 K 2024-04-14T11:17:12,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/cc275a271ab0495a8166ea57ded680ce as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/cc275a271ab0495a8166ea57ded680ce 2024-04-14T11:17:12,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/cc275a271ab0495a8166ea57ded680ce, entries=100, sequenceid=64, filesize=9.4 K 2024-04-14T11:17:12,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/737a5ed110644ebfb39749d5a3362aa2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/737a5ed110644ebfb39749d5a3362aa2 2024-04-14T11:17:12,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/737a5ed110644ebfb39749d5a3362aa2, entries=100, sequenceid=64, filesize=9.4 K 2024-04-14T11:17:12,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 893136536711aed76a1c4f901f8c1645 in 461ms, sequenceid=64, compaction requested=true 2024-04-14T11:17:12,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:12,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:12,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:12,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:12,685 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:17:12,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:12,685 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:17:12,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:12,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:12,687 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50272 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:17:12,687 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/A is initiating minor compaction (all files) 2024-04-14T11:17:12,687 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/A in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:12,687 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:17:12,687 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/bd58560fe8384777b052c75d10d73bb6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/f15d0e26dd92400b9848c8d1e6a7800b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/abbbe152e181417e87539ecf8ceaa8a2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/d6ce1b581ac141fd8296611e951f3008] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=49.1 K 2024-04-14T11:17:12,687 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/B is initiating minor compaction (all files) 2024-04-14T11:17:12,687 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/B in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:12,687 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/939bf8d4a8e347a3a23c7e6e888382a9, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/f271f3503bd149b0a1aa0231df6f75c6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/13c602a016bf4fc7aa7e8f3b609bfee0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/cc275a271ab0495a8166ea57ded680ce] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=44.5 K 2024-04-14T11:17:12,687 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting bd58560fe8384777b052c75d10d73bb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1713093429431 2024-04-14T11:17:12,688 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 939bf8d4a8e347a3a23c7e6e888382a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1713093429431 2024-04-14T11:17:12,688 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting f15d0e26dd92400b9848c8d1e6a7800b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1713093429445 2024-04-14T11:17:12,688 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting f271f3503bd149b0a1aa0231df6f75c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1713093429445 2024-04-14T11:17:12,688 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting abbbe152e181417e87539ecf8ceaa8a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1713093430074 2024-04-14T11:17:12,689 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 13c602a016bf4fc7aa7e8f3b609bfee0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1713093430074 2024-04-14T11:17:12,689 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting d6ce1b581ac141fd8296611e951f3008, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1713093432219 2024-04-14T11:17:12,689 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting cc275a271ab0495a8166ea57ded680ce, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1713093432219 2024-04-14T11:17:12,701 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#A#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:12,701 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/2c5aecdb2ebd41eaaadca15647edc289 is 50, key is test_row_0/A:col10/1713093432222/Put/seqid=0 2024-04-14T11:17:12,705 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#B#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:12,705 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/31a6162cd55747509c7b99eb8a9f909e is 50, key is test_row_0/B:col10/1713093432222/Put/seqid=0 2024-04-14T11:17:12,714 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/2c5aecdb2ebd41eaaadca15647edc289 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2c5aecdb2ebd41eaaadca15647edc289 2024-04-14T11:17:12,716 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/31a6162cd55747509c7b99eb8a9f909e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/31a6162cd55747509c7b99eb8a9f909e 2024-04-14T11:17:12,723 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 893136536711aed76a1c4f901f8c1645/A of 893136536711aed76a1c4f901f8c1645 into 2c5aecdb2ebd41eaaadca15647edc289(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:12,723 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:12,723 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/A, priority=12, startTime=1713093432684; duration=0sec 2024-04-14T11:17:12,723 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:12,723 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:A 2024-04-14T11:17:12,723 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:17:12,724 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 893136536711aed76a1c4f901f8c1645/B of 893136536711aed76a1c4f901f8c1645 into 31a6162cd55747509c7b99eb8a9f909e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:12,724 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:12,724 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/B, priority=12, startTime=1713093432684; duration=0sec 2024-04-14T11:17:12,724 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:12,724 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:B 2024-04-14T11:17:12,725 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:17:12,725 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/C is initiating minor compaction (all files) 2024-04-14T11:17:12,725 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/C in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:12,725 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/69e4b31a0b2a4e75bddaf53cc5263412, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/90ee543f1dab4f9a98372dbfe73d1f2a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/b7975f3e707c44e1a6667ca2ccd785f5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/737a5ed110644ebfb39749d5a3362aa2] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=44.5 K 2024-04-14T11:17:12,727 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 69e4b31a0b2a4e75bddaf53cc5263412, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1713093429431 2024-04-14T11:17:12,727 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 90ee543f1dab4f9a98372dbfe73d1f2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1713093429445 2024-04-14T11:17:12,727 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting b7975f3e707c44e1a6667ca2ccd785f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1713093430074 2024-04-14T11:17:12,728 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 737a5ed110644ebfb39749d5a3362aa2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1713093432219 2024-04-14T11:17:12,738 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#C#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:12,739 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/f8a7ffb1b58e485eb1eccd72d121ab3e is 50, key is test_row_0/C:col10/1713093432222/Put/seqid=0 2024-04-14T11:17:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=140 2024-04-14T11:17:12,753 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/f8a7ffb1b58e485eb1eccd72d121ab3e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/f8a7ffb1b58e485eb1eccd72d121ab3e 2024-04-14T11:17:12,758 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-04-14T11:17:12,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:12,759 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-04-14T11:17:12,761 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 893136536711aed76a1c4f901f8c1645/C of 893136536711aed76a1c4f901f8c1645 into f8a7ffb1b58e485eb1eccd72d121ab3e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:12,761 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:12,761 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/C, priority=12, startTime=1713093432685; duration=0sec 2024-04-14T11:17:12,761 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:12,761 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:C 2024-04-14T11:17:12,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/ac16d7a2d43c48a6b08216c5cc1e38c8 is 50, key is test_row_0/A:col10/1713093432256/Put/seqid=0 2024-04-14T11:17:12,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:12,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:12,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093492889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093492889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093492891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093492893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093492895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093492997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:12,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:12,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093492997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:13,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093492999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:13,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093492999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,170 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/ac16d7a2d43c48a6b08216c5cc1e38c8 2024-04-14T11:17:13,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/e469df02f3974627a24b0296019f0da2 is 50, key is test_row_0/B:col10/1713093432256/Put/seqid=0 2024-04-14T11:17:13,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:13,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093493199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:13,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093493200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:13,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093493205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:13,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093493205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,217 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/e469df02f3974627a24b0296019f0da2 2024-04-14T11:17:13,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/aaa5954f44fd49139b5215c906939e38 is 50, key is test_row_0/C:col10/1713093432256/Put/seqid=0 2024-04-14T11:17:13,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=140 2024-04-14T11:17:13,285 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/aaa5954f44fd49139b5215c906939e38 2024-04-14T11:17:13,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/ac16d7a2d43c48a6b08216c5cc1e38c8 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/ac16d7a2d43c48a6b08216c5cc1e38c8 2024-04-14T11:17:13,299 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/ac16d7a2d43c48a6b08216c5cc1e38c8, entries=150, sequenceid=92, filesize=11.7 K 2024-04-14T11:17:13,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/e469df02f3974627a24b0296019f0da2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/e469df02f3974627a24b0296019f0da2 2024-04-14T11:17:13,305 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/e469df02f3974627a24b0296019f0da2, entries=150, sequenceid=92, filesize=11.7 K 2024-04-14T11:17:13,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/aaa5954f44fd49139b5215c906939e38 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/aaa5954f44fd49139b5215c906939e38 2024-04-14T11:17:13,311 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/aaa5954f44fd49139b5215c906939e38, entries=150, sequenceid=92, filesize=11.7 K 2024-04-14T11:17:13,312 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3035): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 893136536711aed76a1c4f901f8c1645 in 553ms, sequenceid=92, compaction requested=false 2024-04-14T11:17:13,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:13,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:13,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=141 2024-04-14T11:17:13,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=141 2024-04-14T11:17:13,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=141, resume processing ppid=140 2024-04-14T11:17:13,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=141, ppid=140, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1670 sec 2024-04-14T11:17:13,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=140, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 1.1730 sec 2024-04-14T11:17:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:13,400 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:13,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-04-14T11:17:13,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/1ce0f277edcf4a4898bf89089e0c365f is 50, key is test_row_0/A:col10/1713093432891/Put/seqid=0 2024-04-14T11:17:13,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/1ce0f277edcf4a4898bf89089e0c365f 2024-04-14T11:17:13,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/7dee022506ae4f21a932f25cb584f69f is 50, key is test_row_0/B:col10/1713093432891/Put/seqid=0 2024-04-14T11:17:13,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/7dee022506ae4f21a932f25cb584f69f 2024-04-14T11:17:13,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/c26d2ee7507d43e3879909af0748b8c4 is 50, key is test_row_0/C:col10/1713093432891/Put/seqid=0 2024-04-14T11:17:13,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093493512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093493513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093493514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093493516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:13,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093493517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:13,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093493620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093493620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:13,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093493624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093493625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,769 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-04-14T11:17:13,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:13,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093493827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:13,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093493827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:13,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093493831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:13,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093493831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:13,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/c26d2ee7507d43e3879909af0748b8c4 2024-04-14T11:17:13,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/1ce0f277edcf4a4898bf89089e0c365f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/1ce0f277edcf4a4898bf89089e0c365f 2024-04-14T11:17:13,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/1ce0f277edcf4a4898bf89089e0c365f, entries=200, sequenceid=104, filesize=14.0 K 2024-04-14T11:17:13,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/7dee022506ae4f21a932f25cb584f69f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7dee022506ae4f21a932f25cb584f69f 2024-04-14T11:17:13,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7dee022506ae4f21a932f25cb584f69f, entries=150, sequenceid=104, filesize=11.7 K 2024-04-14T11:17:13,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/c26d2ee7507d43e3879909af0748b8c4 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/c26d2ee7507d43e3879909af0748b8c4 2024-04-14T11:17:13,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/c26d2ee7507d43e3879909af0748b8c4, entries=150, sequenceid=104, filesize=11.7 K 2024-04-14T11:17:13,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 893136536711aed76a1c4f901f8c1645 in 474ms, sequenceid=104, compaction requested=true 2024-04-14T11:17:13,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:13,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:13,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:13,875 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:13,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:13,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:13,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:13,875 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:13,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:13,876 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38430 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:13,876 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/A is initiating minor compaction (all files) 2024-04-14T11:17:13,876 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/A in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:13,876 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2c5aecdb2ebd41eaaadca15647edc289, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/ac16d7a2d43c48a6b08216c5cc1e38c8, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/1ce0f277edcf4a4898bf89089e0c365f] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=37.5 K 2024-04-14T11:17:13,876 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36090 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:13,876 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/B is initiating minor compaction (all files) 2024-04-14T11:17:13,876 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/B in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:13,876 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/31a6162cd55747509c7b99eb8a9f909e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/e469df02f3974627a24b0296019f0da2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7dee022506ae4f21a932f25cb584f69f] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=35.2 K 2024-04-14T11:17:13,877 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2c5aecdb2ebd41eaaadca15647edc289, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1713093430081 2024-04-14T11:17:13,877 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 31a6162cd55747509c7b99eb8a9f909e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1713093430081 2024-04-14T11:17:13,878 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting e469df02f3974627a24b0296019f0da2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1713093432256 2024-04-14T11:17:13,878 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting ac16d7a2d43c48a6b08216c5cc1e38c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1713093432256 2024-04-14T11:17:13,878 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 7dee022506ae4f21a932f25cb584f69f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1713093432890 2024-04-14T11:17:13,878 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1ce0f277edcf4a4898bf89089e0c365f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1713093432890 2024-04-14T11:17:13,886 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#B#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:13,886 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#A#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:13,887 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/7324e1bee8494aeeac4ce724f150ced2 is 50, key is test_row_0/B:col10/1713093432891/Put/seqid=0 2024-04-14T11:17:13,887 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/5a0ecc302436459b8289056e6159df05 is 50, key is test_row_0/A:col10/1713093432891/Put/seqid=0 2024-04-14T11:17:13,911 WARN [DataStreamer for file /user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/7324e1bee8494aeeac4ce724f150ced2 {}] hdfs.DataStreamer(982): Caught exception java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:1.8.0_362] at java.lang.Thread.join(Thread.java:1257) ~[?:1.8.0_362] at java.lang.Thread.join(Thread.java:1331) ~[?:1.8.0_362] at org.apache.hadoop.hdfs.DataStreamer.closeResponder(DataStreamer.java:980) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DataStreamer.endBlock(DataStreamer.java:630) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:807) ~[hadoop-hdfs-client-2.10.2.jar:?] 2024-04-14T11:17:13,919 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/5a0ecc302436459b8289056e6159df05 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/5a0ecc302436459b8289056e6159df05 2024-04-14T11:17:13,931 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/7324e1bee8494aeeac4ce724f150ced2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7324e1bee8494aeeac4ce724f150ced2 2024-04-14T11:17:13,932 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/A of 893136536711aed76a1c4f901f8c1645 into 5a0ecc302436459b8289056e6159df05(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:13,932 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:13,932 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/A, priority=13, startTime=1713093433875; duration=0sec 2024-04-14T11:17:13,932 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:13,932 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:A 2024-04-14T11:17:13,932 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:13,934 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36090 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:13,934 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/C is initiating minor compaction (all files) 2024-04-14T11:17:13,935 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/C in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:13,935 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/f8a7ffb1b58e485eb1eccd72d121ab3e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/aaa5954f44fd49139b5215c906939e38, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/c26d2ee7507d43e3879909af0748b8c4] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=35.2 K 2024-04-14T11:17:13,936 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting f8a7ffb1b58e485eb1eccd72d121ab3e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1713093430081 2024-04-14T11:17:13,936 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting aaa5954f44fd49139b5215c906939e38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1713093432256 2024-04-14T11:17:13,936 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c26d2ee7507d43e3879909af0748b8c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1713093432890 2024-04-14T11:17:13,939 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/B of 893136536711aed76a1c4f901f8c1645 into 7324e1bee8494aeeac4ce724f150ced2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:13,939 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:13,939 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/B, priority=13, startTime=1713093433875; duration=0sec 2024-04-14T11:17:13,939 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:13,939 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:B 2024-04-14T11:17:13,946 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#C#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:13,947 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/e9a52fcb3dbe46d7ac433bf8deed11c5 is 50, key is test_row_0/C:col10/1713093432891/Put/seqid=0 2024-04-14T11:17:13,979 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/e9a52fcb3dbe46d7ac433bf8deed11c5 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/e9a52fcb3dbe46d7ac433bf8deed11c5 2024-04-14T11:17:13,985 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/C of 893136536711aed76a1c4f901f8c1645 into e9a52fcb3dbe46d7ac433bf8deed11c5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:13,985 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:13,985 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/C, priority=13, startTime=1713093433875; duration=0sec 2024-04-14T11:17:13,986 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:13,986 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:C 2024-04-14T11:17:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:14,025 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:14,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-04-14T11:17:14,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/2f61fc77ed7048f0b06463d09f487558 is 50, key is test_row_0/A:col10/1713093433512/Put/seqid=0 2024-04-14T11:17:14,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/2f61fc77ed7048f0b06463d09f487558 2024-04-14T11:17:14,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/5db2e1cbe333416d8986c4c9f8bd18e5 is 50, key is test_row_0/B:col10/1713093433512/Put/seqid=0 2024-04-14T11:17:14,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/5db2e1cbe333416d8986c4c9f8bd18e5 2024-04-14T11:17:14,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093494063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/a697061b474141dcbaba423a44764180 is 50, key is test_row_0/C:col10/1713093433512/Put/seqid=0 2024-04-14T11:17:14,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093494131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093494133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093494136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093494137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093494170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=140 2024-04-14T11:17:14,254 INFO [Thread-2284 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 140 completed 2024-04-14T11:17:14,255 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-04-14T11:17:14,259 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=142 2024-04-14T11:17:14,259 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:14,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=142 2024-04-14T11:17:14,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093494376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,411 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-04-14T11:17:14,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:14,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:14,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:14,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(53): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:14,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:14,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/a697061b474141dcbaba423a44764180 2024-04-14T11:17:14,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/2f61fc77ed7048f0b06463d09f487558 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2f61fc77ed7048f0b06463d09f487558 2024-04-14T11:17:14,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2f61fc77ed7048f0b06463d09f487558, entries=200, sequenceid=133, filesize=14.0 K 2024-04-14T11:17:14,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/5db2e1cbe333416d8986c4c9f8bd18e5 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/5db2e1cbe333416d8986c4c9f8bd18e5 2024-04-14T11:17:14,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/5db2e1cbe333416d8986c4c9f8bd18e5, entries=150, sequenceid=133, filesize=11.8 K 2024-04-14T11:17:14,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/a697061b474141dcbaba423a44764180 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a697061b474141dcbaba423a44764180 2024-04-14T11:17:14,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a697061b474141dcbaba423a44764180, entries=150, sequenceid=133, filesize=11.8 K 2024-04-14T11:17:14,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 893136536711aed76a1c4f901f8c1645 in 481ms, sequenceid=133, compaction requested=false 2024-04-14T11:17:14,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:14,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=142 2024-04-14T11:17:14,564 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-04-14T11:17:14,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:14,565 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-04-14T11:17:14,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/020d43d22cd044c281eca1070bf9c60f is 50, key is test_row_0/A:col10/1713093434057/Put/seqid=0 2024-04-14T11:17:14,574 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/020d43d22cd044c281eca1070bf9c60f 2024-04-14T11:17:14,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/b838039d794a4844b49799d8d03410ee is 50, key is test_row_0/B:col10/1713093434057/Put/seqid=0 2024-04-14T11:17:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:14,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:14,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093494667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093494668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093494670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093494672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093494684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093494775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093494775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093494780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093494780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=142 2024-04-14T11:17:14,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093494979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093494979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093494984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093494985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:14,988 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/b838039d794a4844b49799d8d03410ee 2024-04-14T11:17:14,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/a0b9a18969aa4015b47538d9be5123a3 is 50, key is test_row_0/C:col10/1713093434057/Put/seqid=0 2024-04-14T11:17:15,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093495193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:15,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093495284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:15,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093495284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:15,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093495288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:15,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093495289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=142 2024-04-14T11:17:15,402 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/a0b9a18969aa4015b47538d9be5123a3 2024-04-14T11:17:15,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/020d43d22cd044c281eca1070bf9c60f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/020d43d22cd044c281eca1070bf9c60f 2024-04-14T11:17:15,413 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/020d43d22cd044c281eca1070bf9c60f, entries=150, sequenceid=143, filesize=11.8 K 2024-04-14T11:17:15,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/b838039d794a4844b49799d8d03410ee as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b838039d794a4844b49799d8d03410ee 2024-04-14T11:17:15,419 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b838039d794a4844b49799d8d03410ee, entries=150, sequenceid=143, filesize=11.8 K 2024-04-14T11:17:15,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/a0b9a18969aa4015b47538d9be5123a3 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a0b9a18969aa4015b47538d9be5123a3 2024-04-14T11:17:15,425 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a0b9a18969aa4015b47538d9be5123a3, entries=150, sequenceid=143, filesize=11.8 K 2024-04-14T11:17:15,426 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3035): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 893136536711aed76a1c4f901f8c1645 in 861ms, sequenceid=143, compaction requested=true 2024-04-14T11:17:15,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:15,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:15,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=143 2024-04-14T11:17:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=143 2024-04-14T11:17:15,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=143, resume processing ppid=142 2024-04-14T11:17:15,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=143, ppid=142, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1690 sec 2024-04-14T11:17:15,431 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=142, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 1.1750 sec 2024-04-14T11:17:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:15,792 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:15,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-04-14T11:17:15,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/e278f8623c99486ea3e539c78e8d938d is 50, key is test_row_0/A:col10/1713093434670/Put/seqid=0 2024-04-14T11:17:15,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:15,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093495799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:15,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:15,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093495806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:15,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:15,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093495807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:15,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:15,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093495808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:15,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093495908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:15,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:15,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093495911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:15,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093495913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:15,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093495913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:16,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093496112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:16,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:16,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093496116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:16,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:16,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093496119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:16,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:16,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093496119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:16,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/e278f8623c99486ea3e539c78e8d938d 2024-04-14T11:17:16,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093496200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:16,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/7773d08fe416477e8d46c10c0c4d5665 is 50, key is test_row_0/B:col10/1713093434670/Put/seqid=0 2024-04-14T11:17:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=142 2024-04-14T11:17:16,365 INFO [Thread-2284 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 142 completed 2024-04-14T11:17:16,366 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-04-14T11:17:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=144 2024-04-14T11:17:16,370 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:16,371 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:16,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:16,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093496416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:16,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093496419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:16,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093496423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:16,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093496424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=144 2024-04-14T11:17:16,523 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:16,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-04-14T11:17:16,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:16,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:16,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:16,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(53): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:16,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:16,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/7773d08fe416477e8d46c10c0c4d5665 2024-04-14T11:17:16,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/0b1ddf82ac0c4737bf920a990a1c81e5 is 50, key is test_row_0/C:col10/1713093434670/Put/seqid=0 2024-04-14T11:17:16,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/0b1ddf82ac0c4737bf920a990a1c81e5 2024-04-14T11:17:16,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/e278f8623c99486ea3e539c78e8d938d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/e278f8623c99486ea3e539c78e8d938d 2024-04-14T11:17:16,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/e278f8623c99486ea3e539c78e8d938d, entries=200, sequenceid=170, filesize=14.2 K 2024-04-14T11:17:16,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/7773d08fe416477e8d46c10c0c4d5665 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7773d08fe416477e8d46c10c0c4d5665 2024-04-14T11:17:16,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7773d08fe416477e8d46c10c0c4d5665, entries=150, sequenceid=170, filesize=11.8 K 2024-04-14T11:17:16,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/0b1ddf82ac0c4737bf920a990a1c81e5 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/0b1ddf82ac0c4737bf920a990a1c81e5 2024-04-14T11:17:16,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/0b1ddf82ac0c4737bf920a990a1c81e5, entries=150, sequenceid=170, filesize=11.8 K 2024-04-14T11:17:16,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 893136536711aed76a1c4f901f8c1645 in 873ms, sequenceid=170, compaction requested=true 2024-04-14T11:17:16,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:16,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:16,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:16,665 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:17:16,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:16,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:16,665 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:17:16,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:16,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:16,667 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48526 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:17:16,667 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/B is initiating minor compaction (all files) 2024-04-14T11:17:16,667 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53256 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:17:16,667 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/B in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:16,667 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/A is initiating minor compaction (all files) 2024-04-14T11:17:16,668 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/A in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:16,668 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7324e1bee8494aeeac4ce724f150ced2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/5db2e1cbe333416d8986c4c9f8bd18e5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b838039d794a4844b49799d8d03410ee, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7773d08fe416477e8d46c10c0c4d5665] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=47.4 K 2024-04-14T11:17:16,668 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/5a0ecc302436459b8289056e6159df05, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2f61fc77ed7048f0b06463d09f487558, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/020d43d22cd044c281eca1070bf9c60f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/e278f8623c99486ea3e539c78e8d938d] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=52.0 K 2024-04-14T11:17:16,668 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 7324e1bee8494aeeac4ce724f150ced2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1713093432890 2024-04-14T11:17:16,668 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5a0ecc302436459b8289056e6159df05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1713093432890 2024-04-14T11:17:16,668 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 5db2e1cbe333416d8986c4c9f8bd18e5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1713093433480 2024-04-14T11:17:16,669 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2f61fc77ed7048f0b06463d09f487558, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1713093433480 2024-04-14T11:17:16,669 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting b838039d794a4844b49799d8d03410ee, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1713093434052 2024-04-14T11:17:16,669 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 020d43d22cd044c281eca1070bf9c60f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1713093434052 2024-04-14T11:17:16,670 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 7773d08fe416477e8d46c10c0c4d5665, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1713093434667 2024-04-14T11:17:16,670 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting e278f8623c99486ea3e539c78e8d938d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1713093434666 2024-04-14T11:17:16,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=144 2024-04-14T11:17:16,676 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:16,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-04-14T11:17:16,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:16,677 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-04-14T11:17:16,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/a1faeb1293c54374b60ed17ace5c9d78 is 50, key is test_row_0/A:col10/1713093435804/Put/seqid=0 2024-04-14T11:17:16,685 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#A#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:16,686 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/7ef43f1993d943fea215db8d234ce6d0 is 50, key is test_row_0/A:col10/1713093434670/Put/seqid=0 2024-04-14T11:17:16,686 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#B#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:16,686 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/dc49ec4b4ffb4c27a6f0ce9df98401a6 is 50, key is test_row_0/B:col10/1713093434670/Put/seqid=0 2024-04-14T11:17:16,698 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/a1faeb1293c54374b60ed17ace5c9d78 2024-04-14T11:17:16,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/929d423bb7214f6d8e60458248b7286d is 50, key is test_row_0/B:col10/1713093435804/Put/seqid=0 2024-04-14T11:17:16,715 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/7ef43f1993d943fea215db8d234ce6d0 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/7ef43f1993d943fea215db8d234ce6d0 2024-04-14T11:17:16,721 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/929d423bb7214f6d8e60458248b7286d 2024-04-14T11:17:16,722 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 893136536711aed76a1c4f901f8c1645/A of 893136536711aed76a1c4f901f8c1645 into 7ef43f1993d943fea215db8d234ce6d0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:16,723 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:16,723 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/A, priority=12, startTime=1713093436665; duration=0sec 2024-04-14T11:17:16,723 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:16,723 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:A 2024-04-14T11:17:16,723 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:17:16,726 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48526 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:17:16,726 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/C is initiating minor compaction (all files) 2024-04-14T11:17:16,726 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/C in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:16,726 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/e9a52fcb3dbe46d7ac433bf8deed11c5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a697061b474141dcbaba423a44764180, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a0b9a18969aa4015b47538d9be5123a3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/0b1ddf82ac0c4737bf920a990a1c81e5] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=47.4 K 2024-04-14T11:17:16,726 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting e9a52fcb3dbe46d7ac433bf8deed11c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1713093432890 2024-04-14T11:17:16,727 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting a697061b474141dcbaba423a44764180, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1713093433480 2024-04-14T11:17:16,727 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting a0b9a18969aa4015b47538d9be5123a3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1713093434052 2024-04-14T11:17:16,728 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 0b1ddf82ac0c4737bf920a990a1c81e5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1713093434667 2024-04-14T11:17:16,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/f24272593a654670ad088bece4da203f is 50, key is test_row_0/C:col10/1713093435804/Put/seqid=0 2024-04-14T11:17:16,735 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/f24272593a654670ad088bece4da203f 2024-04-14T11:17:16,737 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#C#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:16,737 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/facfcb2cd4c645bf9a982a61740a2228 is 50, key is test_row_0/C:col10/1713093434670/Put/seqid=0 2024-04-14T11:17:16,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/a1faeb1293c54374b60ed17ace5c9d78 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/a1faeb1293c54374b60ed17ace5c9d78 2024-04-14T11:17:16,747 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/a1faeb1293c54374b60ed17ace5c9d78, entries=100, sequenceid=180, filesize=9.5 K 2024-04-14T11:17:16,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/929d423bb7214f6d8e60458248b7286d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/929d423bb7214f6d8e60458248b7286d 2024-04-14T11:17:16,749 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/facfcb2cd4c645bf9a982a61740a2228 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/facfcb2cd4c645bf9a982a61740a2228 2024-04-14T11:17:16,753 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/929d423bb7214f6d8e60458248b7286d, entries=100, sequenceid=180, filesize=9.5 K 2024-04-14T11:17:16,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/f24272593a654670ad088bece4da203f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/f24272593a654670ad088bece4da203f 2024-04-14T11:17:16,755 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 893136536711aed76a1c4f901f8c1645/C of 893136536711aed76a1c4f901f8c1645 into facfcb2cd4c645bf9a982a61740a2228(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:16,755 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:16,755 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/C, priority=12, startTime=1713093436665; duration=0sec 2024-04-14T11:17:16,755 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:16,755 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:C 2024-04-14T11:17:16,758 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/f24272593a654670ad088bece4da203f, entries=100, sequenceid=180, filesize=9.5 K 2024-04-14T11:17:16,759 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3035): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 893136536711aed76a1c4f901f8c1645 in 82ms, sequenceid=180, compaction requested=false 2024-04-14T11:17:16,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:16,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:16,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=145 2024-04-14T11:17:16,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=145 2024-04-14T11:17:16,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=145, resume processing ppid=144 2024-04-14T11:17:16,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=145, ppid=144, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 389 msec 2024-04-14T11:17:16,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=144, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 395 msec 2024-04-14T11:17:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:16,934 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:16,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-04-14T11:17:16,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/a93d3d7adb504c4ba55c1a184f37de3c is 50, key is test_row_0/A:col10/1713093436933/Put/seqid=0 2024-04-14T11:17:16,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=144 2024-04-14T11:17:16,974 INFO [Thread-2284 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 144 completed 2024-04-14T11:17:16,976 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-04-14T11:17:16,979 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=146 2024-04-14T11:17:16,980 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:16,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:17,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093497003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093497003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093497004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093497004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=146 2024-04-14T11:17:17,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093497111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093497111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093497111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,115 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/dc49ec4b4ffb4c27a6f0ce9df98401a6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/dc49ec4b4ffb4c27a6f0ce9df98401a6 2024-04-14T11:17:17,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:17,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093497111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,121 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 893136536711aed76a1c4f901f8c1645/B of 893136536711aed76a1c4f901f8c1645 into dc49ec4b4ffb4c27a6f0ce9df98401a6(size=12.2 K), total size for store is 21.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:17,121 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:17,121 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/B, priority=12, startTime=1713093436665; duration=0sec 2024-04-14T11:17:17,121 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:17,121 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:B 2024-04-14T11:17:17,131 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-04-14T11:17:17,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:17,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:17,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:17,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(53): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:17,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=146 2024-04-14T11:17:17,285 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-04-14T11:17:17,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:17,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:17,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:17,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(53): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:17,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:17,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:17,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:17,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093497315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:17,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093497315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093497316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:17,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093497317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/a93d3d7adb504c4ba55c1a184f37de3c 2024-04-14T11:17:17,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/0f9ad6055d4d48caa08a358d4666b954 is 50, key is test_row_0/B:col10/1713093436933/Put/seqid=0 2024-04-14T11:17:17,438 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-04-14T11:17:17,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:17,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:17,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:17,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(53): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:17,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:17,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:17,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=146 2024-04-14T11:17:17,621 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-04-14T11:17:17,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:17,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:17,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:17,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(53): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:17,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:17,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093497619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:17,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093497619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:17,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093497620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:17,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093497625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/0f9ad6055d4d48caa08a358d4666b954 2024-04-14T11:17:17,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/e0fa6f68445e467da5fc844a9e4def02 is 50, key is test_row_0/C:col10/1713093436933/Put/seqid=0 2024-04-14T11:17:17,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/e0fa6f68445e467da5fc844a9e4def02 2024-04-14T11:17:17,776 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-04-14T11:17:17,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:17,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:17,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:17,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(53): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:17,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:17,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/a93d3d7adb504c4ba55c1a184f37de3c as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/a93d3d7adb504c4ba55c1a184f37de3c 2024-04-14T11:17:17,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:17,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/a93d3d7adb504c4ba55c1a184f37de3c, entries=300, sequenceid=193, filesize=18.9 K 2024-04-14T11:17:17,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/0f9ad6055d4d48caa08a358d4666b954 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/0f9ad6055d4d48caa08a358d4666b954 2024-04-14T11:17:17,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/0f9ad6055d4d48caa08a358d4666b954, entries=150, sequenceid=193, filesize=11.8 K 2024-04-14T11:17:17,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/e0fa6f68445e467da5fc844a9e4def02 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/e0fa6f68445e467da5fc844a9e4def02 2024-04-14T11:17:17,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/e0fa6f68445e467da5fc844a9e4def02, entries=150, sequenceid=193, filesize=11.8 K 2024-04-14T11:17:17,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 893136536711aed76a1c4f901f8c1645 in 862ms, sequenceid=193, compaction requested=true 2024-04-14T11:17:17,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:17,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:17,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:17,796 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:17,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:17,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:17,796 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:17,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:17,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:17,798 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41554 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:17,799 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/A is initiating minor compaction (all files) 2024-04-14T11:17:17,799 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/A in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:17,799 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34384 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:17,799 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/B is initiating minor compaction (all files) 2024-04-14T11:17:17,799 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/7ef43f1993d943fea215db8d234ce6d0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/a1faeb1293c54374b60ed17ace5c9d78, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/a93d3d7adb504c4ba55c1a184f37de3c] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=40.6 K 2024-04-14T11:17:17,799 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/B in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:17,799 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/dc49ec4b4ffb4c27a6f0ce9df98401a6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/929d423bb7214f6d8e60458248b7286d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/0f9ad6055d4d48caa08a358d4666b954] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=33.6 K 2024-04-14T11:17:17,806 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7ef43f1993d943fea215db8d234ce6d0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1713093434667 2024-04-14T11:17:17,806 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting a1faeb1293c54374b60ed17ace5c9d78, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1713093435804 2024-04-14T11:17:17,812 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting dc49ec4b4ffb4c27a6f0ce9df98401a6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1713093434667 2024-04-14T11:17:17,813 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting a93d3d7adb504c4ba55c1a184f37de3c, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1713093436926 2024-04-14T11:17:17,813 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 929d423bb7214f6d8e60458248b7286d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1713093435804 2024-04-14T11:17:17,814 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 0f9ad6055d4d48caa08a358d4666b954, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1713093436928 2024-04-14T11:17:17,829 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#A#compaction#478 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:17,830 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#B#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:17,830 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/b660bd91441c48a896894ab650602d81 is 50, key is test_row_0/A:col10/1713093436933/Put/seqid=0 2024-04-14T11:17:17,830 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/4ed93992f061433298ae43e3008c84bf is 50, key is test_row_0/B:col10/1713093436933/Put/seqid=0 2024-04-14T11:17:17,843 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/b660bd91441c48a896894ab650602d81 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/b660bd91441c48a896894ab650602d81 2024-04-14T11:17:17,846 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/4ed93992f061433298ae43e3008c84bf as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/4ed93992f061433298ae43e3008c84bf 2024-04-14T11:17:17,849 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/A of 893136536711aed76a1c4f901f8c1645 into b660bd91441c48a896894ab650602d81(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:17,849 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:17,849 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/A, priority=13, startTime=1713093437796; duration=0sec 2024-04-14T11:17:17,850 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:17,850 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:A 2024-04-14T11:17:17,850 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:17,851 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34384 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:17,852 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/C is initiating minor compaction (all files) 2024-04-14T11:17:17,852 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/C in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:17,852 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/facfcb2cd4c645bf9a982a61740a2228, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/f24272593a654670ad088bece4da203f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/e0fa6f68445e467da5fc844a9e4def02] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=33.6 K 2024-04-14T11:17:17,853 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting facfcb2cd4c645bf9a982a61740a2228, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1713093434667 2024-04-14T11:17:17,853 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting f24272593a654670ad088bece4da203f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1713093435804 2024-04-14T11:17:17,853 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/B of 893136536711aed76a1c4f901f8c1645 into 4ed93992f061433298ae43e3008c84bf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:17,853 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:17,853 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/B, priority=13, startTime=1713093437796; duration=0sec 2024-04-14T11:17:17,853 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:17,853 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:B 2024-04-14T11:17:17,853 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting e0fa6f68445e467da5fc844a9e4def02, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1713093436928 2024-04-14T11:17:17,860 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#C#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:17,861 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/836380589c254291917c8b5a2b90caea is 50, key is test_row_0/C:col10/1713093436933/Put/seqid=0 2024-04-14T11:17:17,871 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/836380589c254291917c8b5a2b90caea as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/836380589c254291917c8b5a2b90caea 2024-04-14T11:17:17,876 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/C of 893136536711aed76a1c4f901f8c1645 into 836380589c254291917c8b5a2b90caea(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:17,876 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:17,877 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/C, priority=13, startTime=1713093437796; duration=0sec 2024-04-14T11:17:17,877 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:17,877 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:C 2024-04-14T11:17:17,929 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:17,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-04-14T11:17:17,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:17,930 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-04-14T11:17:17,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/fd4b9ee342334010a33c2afdd0911c4a is 50, key is test_row_0/A:col10/1713093437001/Put/seqid=0 2024-04-14T11:17:17,940 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/fd4b9ee342334010a33c2afdd0911c4a 2024-04-14T11:17:17,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/c24dc2ebbc1c4664bc91f8908dd332b3 is 50, key is test_row_0/B:col10/1713093437001/Put/seqid=0 2024-04-14T11:17:17,954 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/c24dc2ebbc1c4664bc91f8908dd332b3 2024-04-14T11:17:17,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/5c0c5734db02480fb43ca9296b8002fe is 50, key is test_row_0/C:col10/1713093437001/Put/seqid=0 2024-04-14T11:17:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=146 2024-04-14T11:17:18,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:18,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093498131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093498132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093498135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093498136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093498214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,218 DEBUG [Thread-2280 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4153 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., hostname=jenkins-hbase6.apache.org,33239,1713093321027, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2024-04-14T11:17:18,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093498237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093498237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093498239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093498240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,371 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/5c0c5734db02480fb43ca9296b8002fe 2024-04-14T11:17:18,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/fd4b9ee342334010a33c2afdd0911c4a as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/fd4b9ee342334010a33c2afdd0911c4a 2024-04-14T11:17:18,382 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/fd4b9ee342334010a33c2afdd0911c4a, entries=150, sequenceid=223, filesize=11.8 K 2024-04-14T11:17:18,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/c24dc2ebbc1c4664bc91f8908dd332b3 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/c24dc2ebbc1c4664bc91f8908dd332b3 2024-04-14T11:17:18,389 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/c24dc2ebbc1c4664bc91f8908dd332b3, entries=150, sequenceid=223, filesize=11.8 K 2024-04-14T11:17:18,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/5c0c5734db02480fb43ca9296b8002fe as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/5c0c5734db02480fb43ca9296b8002fe 2024-04-14T11:17:18,395 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/5c0c5734db02480fb43ca9296b8002fe, entries=150, sequenceid=223, filesize=11.8 K 2024-04-14T11:17:18,396 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3035): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 893136536711aed76a1c4f901f8c1645 in 466ms, sequenceid=223, compaction requested=false 2024-04-14T11:17:18,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:18,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:18,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=147 2024-04-14T11:17:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=147 2024-04-14T11:17:18,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=147, resume processing ppid=146 2024-04-14T11:17:18,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=147, ppid=146, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4170 sec 2024-04-14T11:17:18,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=146, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.4230 sec 2024-04-14T11:17:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:18,445 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:18,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-04-14T11:17:18,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/e5918d0372944e6fb27e86d9c92555a0 is 50, key is test_row_0/A:col10/1713093438445/Put/seqid=0 2024-04-14T11:17:18,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093498468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093498473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093498472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093498473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093498574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093498582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093498582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093498582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093498779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093498785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093498787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093498787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:18,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/e5918d0372944e6fb27e86d9c92555a0 2024-04-14T11:17:18,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/c970a67fa6664bf393938b751e58be72 is 50, key is test_row_0/B:col10/1713093438445/Put/seqid=0 2024-04-14T11:17:18,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/c970a67fa6664bf393938b751e58be72 2024-04-14T11:17:18,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/04d4a39ae219405ebe7930d809130301 is 50, key is test_row_0/C:col10/1713093438445/Put/seqid=0 2024-04-14T11:17:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=146 2024-04-14T11:17:19,085 INFO [Thread-2284 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 146 completed 2024-04-14T11:17:19,086 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-04-14T11:17:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=148 2024-04-14T11:17:19,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:19,090 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093499085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:19,091 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:19,091 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:19,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093499090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:19,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093499091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:19,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093499093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=148 2024-04-14T11:17:19,243 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:19,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-04-14T11:17:19,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:19,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:19,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:19,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(53): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:19,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:19,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/04d4a39ae219405ebe7930d809130301 2024-04-14T11:17:19,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/e5918d0372944e6fb27e86d9c92555a0 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/e5918d0372944e6fb27e86d9c92555a0 2024-04-14T11:17:19,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/e5918d0372944e6fb27e86d9c92555a0, entries=250, sequenceid=236, filesize=16.5 K 2024-04-14T11:17:19,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/c970a67fa6664bf393938b751e58be72 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/c970a67fa6664bf393938b751e58be72 2024-04-14T11:17:19,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/c970a67fa6664bf393938b751e58be72, entries=150, sequenceid=236, filesize=11.8 K 2024-04-14T11:17:19,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/04d4a39ae219405ebe7930d809130301 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/04d4a39ae219405ebe7930d809130301 2024-04-14T11:17:19,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/04d4a39ae219405ebe7930d809130301, entries=150, sequenceid=236, filesize=11.8 K 2024-04-14T11:17:19,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 893136536711aed76a1c4f901f8c1645 in 863ms, sequenceid=236, compaction requested=true 2024-04-14T11:17:19,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:19,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:19,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:19,309 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:19,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:19,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:19,309 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:19,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:19,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:19,311 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41660 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:19,311 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/A is initiating minor compaction (all files) 2024-04-14T11:17:19,311 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/A in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:19,311 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/b660bd91441c48a896894ab650602d81, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/fd4b9ee342334010a33c2afdd0911c4a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/e5918d0372944e6fb27e86d9c92555a0] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=40.7 K 2024-04-14T11:17:19,312 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36880 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:19,312 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting b660bd91441c48a896894ab650602d81, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1713093436928 2024-04-14T11:17:19,312 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/B is initiating minor compaction (all files) 2024-04-14T11:17:19,312 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/B in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:19,312 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/4ed93992f061433298ae43e3008c84bf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/c24dc2ebbc1c4664bc91f8908dd332b3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/c970a67fa6664bf393938b751e58be72] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=36.0 K 2024-04-14T11:17:19,313 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting fd4b9ee342334010a33c2afdd0911c4a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1713093437001 2024-04-14T11:17:19,313 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 4ed93992f061433298ae43e3008c84bf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1713093436928 2024-04-14T11:17:19,313 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting e5918d0372944e6fb27e86d9c92555a0, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1713093438126 2024-04-14T11:17:19,314 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting c24dc2ebbc1c4664bc91f8908dd332b3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1713093437001 2024-04-14T11:17:19,314 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting c970a67fa6664bf393938b751e58be72, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1713093438134 2024-04-14T11:17:19,322 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#A#compaction#487 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:19,323 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/de38e29978b04e12a6457ece2adf1e6d is 50, key is test_row_0/A:col10/1713093438445/Put/seqid=0 2024-04-14T11:17:19,326 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#B#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:19,326 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/a838fc6c87be441483f9ed601ceb0525 is 50, key is test_row_0/B:col10/1713093438445/Put/seqid=0 2024-04-14T11:17:19,338 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/de38e29978b04e12a6457ece2adf1e6d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/de38e29978b04e12a6457ece2adf1e6d 2024-04-14T11:17:19,347 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/A of 893136536711aed76a1c4f901f8c1645 into de38e29978b04e12a6457ece2adf1e6d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:19,347 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:19,347 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/A, priority=13, startTime=1713093439309; duration=0sec 2024-04-14T11:17:19,347 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:19,347 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:A 2024-04-14T11:17:19,347 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:19,347 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/a838fc6c87be441483f9ed601ceb0525 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/a838fc6c87be441483f9ed601ceb0525 2024-04-14T11:17:19,349 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36880 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:19,349 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/C is initiating minor compaction (all files) 2024-04-14T11:17:19,349 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/C in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:19,349 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/836380589c254291917c8b5a2b90caea, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/5c0c5734db02480fb43ca9296b8002fe, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/04d4a39ae219405ebe7930d809130301] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=36.0 K 2024-04-14T11:17:19,350 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 836380589c254291917c8b5a2b90caea, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1713093436928 2024-04-14T11:17:19,350 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5c0c5734db02480fb43ca9296b8002fe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1713093437001 2024-04-14T11:17:19,351 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 04d4a39ae219405ebe7930d809130301, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1713093438134 2024-04-14T11:17:19,357 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/B of 893136536711aed76a1c4f901f8c1645 into a838fc6c87be441483f9ed601ceb0525(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:19,357 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:19,357 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/B, priority=13, startTime=1713093439309; duration=0sec 2024-04-14T11:17:19,357 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:19,357 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:B 2024-04-14T11:17:19,358 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#C#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:19,359 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/778bdeaae8c3448c8d65dd914fdf5311 is 50, key is test_row_0/C:col10/1713093438445/Put/seqid=0 2024-04-14T11:17:19,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=148 2024-04-14T11:17:19,397 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:19,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-04-14T11:17:19,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:19,397 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-04-14T11:17:19,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/cc1a6a03a37d484e98e70bead0af4877 is 50, key is test_row_0/A:col10/1713093438472/Put/seqid=0 2024-04-14T11:17:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:19,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:19,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:19,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093499607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:19,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093499611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:19,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093499620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:19,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:19,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093499621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=148 2024-04-14T11:17:19,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:19,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093499723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:19,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:19,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093499725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:19,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:19,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093499725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:19,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093499727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:19,771 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/778bdeaae8c3448c8d65dd914fdf5311 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/778bdeaae8c3448c8d65dd914fdf5311 2024-04-14T11:17:19,779 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/C of 893136536711aed76a1c4f901f8c1645 into 778bdeaae8c3448c8d65dd914fdf5311(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:19,779 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:19,779 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/C, priority=13, startTime=1713093439309; duration=0sec 2024-04-14T11:17:19,779 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:19,779 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:C 2024-04-14T11:17:19,807 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/cc1a6a03a37d484e98e70bead0af4877 2024-04-14T11:17:19,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/0b3ef9c50f4045a590b0506529207e31 is 50, key is test_row_0/B:col10/1713093438472/Put/seqid=0 2024-04-14T11:17:19,832 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/0b3ef9c50f4045a590b0506529207e31 2024-04-14T11:17:19,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/72253e6f5ab94ba1a0563e911d9897cf is 50, key is test_row_0/C:col10/1713093438472/Put/seqid=0 2024-04-14T11:17:19,858 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/72253e6f5ab94ba1a0563e911d9897cf 2024-04-14T11:17:19,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/cc1a6a03a37d484e98e70bead0af4877 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/cc1a6a03a37d484e98e70bead0af4877 2024-04-14T11:17:19,874 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/cc1a6a03a37d484e98e70bead0af4877, entries=150, sequenceid=261, filesize=11.9 K 2024-04-14T11:17:19,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/0b3ef9c50f4045a590b0506529207e31 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/0b3ef9c50f4045a590b0506529207e31 2024-04-14T11:17:19,880 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/0b3ef9c50f4045a590b0506529207e31, entries=150, sequenceid=261, filesize=11.9 K 2024-04-14T11:17:19,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/72253e6f5ab94ba1a0563e911d9897cf as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/72253e6f5ab94ba1a0563e911d9897cf 2024-04-14T11:17:19,894 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/72253e6f5ab94ba1a0563e911d9897cf, entries=150, sequenceid=261, filesize=11.9 K 2024-04-14T11:17:19,895 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3035): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 893136536711aed76a1c4f901f8c1645 in 498ms, sequenceid=261, compaction requested=false 2024-04-14T11:17:19,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:19,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:19,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=149 2024-04-14T11:17:19,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=149 2024-04-14T11:17:19,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=149, resume processing ppid=148 2024-04-14T11:17:19,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=149, ppid=148, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 805 msec 2024-04-14T11:17:19,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=148, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 813 msec 2024-04-14T11:17:19,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:19,930 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:19,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-04-14T11:17:19,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/686e30358dcf4eb9934550624252799d is 50, key is test_row_0/A:col10/1713093439612/Put/seqid=0 2024-04-14T11:17:19,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/686e30358dcf4eb9934550624252799d 2024-04-14T11:17:19,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/2109eed54f904a8c8991af8f34325193 is 50, key is test_row_0/B:col10/1713093439612/Put/seqid=0 2024-04-14T11:17:19,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/2109eed54f904a8c8991af8f34325193 2024-04-14T11:17:19,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/5c05ccef6c5043c4992b8510f6b78e13 is 50, key is test_row_0/C:col10/1713093439612/Put/seqid=0 2024-04-14T11:17:19,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:19,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093499960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:19,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:19,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093499960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:19,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:19,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093499961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:19,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093499962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:19,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/5c05ccef6c5043c4992b8510f6b78e13 2024-04-14T11:17:19,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/686e30358dcf4eb9934550624252799d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/686e30358dcf4eb9934550624252799d 2024-04-14T11:17:19,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/686e30358dcf4eb9934550624252799d, entries=250, sequenceid=277, filesize=16.8 K 2024-04-14T11:17:19,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/2109eed54f904a8c8991af8f34325193 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/2109eed54f904a8c8991af8f34325193 2024-04-14T11:17:19,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/2109eed54f904a8c8991af8f34325193, entries=150, sequenceid=277, filesize=12.0 K 2024-04-14T11:17:19,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/5c05ccef6c5043c4992b8510f6b78e13 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/5c05ccef6c5043c4992b8510f6b78e13 2024-04-14T11:17:19,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/5c05ccef6c5043c4992b8510f6b78e13, entries=150, sequenceid=277, filesize=12.0 K 2024-04-14T11:17:19,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 893136536711aed76a1c4f901f8c1645 in 63ms, sequenceid=277, compaction requested=true 2024-04-14T11:17:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:19,993 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:19,993 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:19,994 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42112 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:19,994 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/A is initiating minor compaction (all files) 2024-04-14T11:17:19,995 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/A in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:19,995 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/de38e29978b04e12a6457ece2adf1e6d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/cc1a6a03a37d484e98e70bead0af4877, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/686e30358dcf4eb9934550624252799d] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=41.1 K 2024-04-14T11:17:19,995 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37232 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:19,995 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/B is initiating minor compaction (all files) 2024-04-14T11:17:19,995 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/B in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:19,995 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/a838fc6c87be441483f9ed601ceb0525, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/0b3ef9c50f4045a590b0506529207e31, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/2109eed54f904a8c8991af8f34325193] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=36.4 K 2024-04-14T11:17:19,995 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting de38e29978b04e12a6457ece2adf1e6d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1713093438134 2024-04-14T11:17:19,996 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting a838fc6c87be441483f9ed601ceb0525, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1713093438134 2024-04-14T11:17:19,996 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting cc1a6a03a37d484e98e70bead0af4877, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1713093438471 2024-04-14T11:17:19,996 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 0b3ef9c50f4045a590b0506529207e31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1713093438471 2024-04-14T11:17:19,996 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 686e30358dcf4eb9934550624252799d, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1713093439606 2024-04-14T11:17:19,997 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 2109eed54f904a8c8991af8f34325193, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1713093439606 2024-04-14T11:17:20,006 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#A#compaction#496 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:20,006 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/2d3f47b9c8bd484cb4a5c1bd79b00e45 is 50, key is test_row_0/A:col10/1713093439612/Put/seqid=0 2024-04-14T11:17:20,007 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#B#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:20,008 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/bd95ff9ceb6148c5a93338dff5c2a3ec is 50, key is test_row_0/B:col10/1713093439612/Put/seqid=0 2024-04-14T11:17:20,023 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/2d3f47b9c8bd484cb4a5c1bd79b00e45 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2d3f47b9c8bd484cb4a5c1bd79b00e45 2024-04-14T11:17:20,027 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/bd95ff9ceb6148c5a93338dff5c2a3ec as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/bd95ff9ceb6148c5a93338dff5c2a3ec 2024-04-14T11:17:20,031 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/A of 893136536711aed76a1c4f901f8c1645 into 2d3f47b9c8bd484cb4a5c1bd79b00e45(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:20,031 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:20,032 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/A, priority=13, startTime=1713093439993; duration=0sec 2024-04-14T11:17:20,032 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:20,032 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:A 2024-04-14T11:17:20,032 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:20,034 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37232 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:20,034 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/C is initiating minor compaction (all files) 2024-04-14T11:17:20,034 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/C in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:20,034 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/778bdeaae8c3448c8d65dd914fdf5311, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/72253e6f5ab94ba1a0563e911d9897cf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/5c05ccef6c5043c4992b8510f6b78e13] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=36.4 K 2024-04-14T11:17:20,035 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 778bdeaae8c3448c8d65dd914fdf5311, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1713093438134 2024-04-14T11:17:20,036 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 72253e6f5ab94ba1a0563e911d9897cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1713093438471 2024-04-14T11:17:20,036 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/B of 893136536711aed76a1c4f901f8c1645 into bd95ff9ceb6148c5a93338dff5c2a3ec(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:20,036 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:20,036 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/B, priority=13, startTime=1713093439993; duration=0sec 2024-04-14T11:17:20,036 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:20,036 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:B 2024-04-14T11:17:20,037 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5c05ccef6c5043c4992b8510f6b78e13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1713093439606 2024-04-14T11:17:20,045 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#C#compaction#498 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:20,046 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/8d29d736fbe24798bd6f45882f203370 is 50, key is test_row_0/C:col10/1713093439612/Put/seqid=0 2024-04-14T11:17:20,057 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/8d29d736fbe24798bd6f45882f203370 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/8d29d736fbe24798bd6f45882f203370 2024-04-14T11:17:20,064 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/C of 893136536711aed76a1c4f901f8c1645 into 8d29d736fbe24798bd6f45882f203370(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:20,064 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:20,064 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/C, priority=13, startTime=1713093439993; duration=0sec 2024-04-14T11:17:20,064 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:20,064 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:C 2024-04-14T11:17:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:20,071 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:20,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-04-14T11:17:20,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/2e52403eed5c40ff840867d0f1e7e4fe is 50, key is test_row_0/A:col10/1713093439943/Put/seqid=0 2024-04-14T11:17:20,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/2e52403eed5c40ff840867d0f1e7e4fe 2024-04-14T11:17:20,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093500085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093500087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/db81a169aa1e4e1383103d56042ff2bd is 50, key is test_row_0/B:col10/1713093439943/Put/seqid=0 2024-04-14T11:17:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093500090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093500090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/db81a169aa1e4e1383103d56042ff2bd 2024-04-14T11:17:20,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/14380760189c4ff29c33dac91f3bc2e1 is 50, key is test_row_0/C:col10/1713093439943/Put/seqid=0 2024-04-14T11:17:20,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=148 2024-04-14T11:17:20,194 INFO [Thread-2284 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 148 completed 2024-04-14T11:17:20,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093500191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093500191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093500192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,196 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:20,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-04-14T11:17:20,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=150 2024-04-14T11:17:20,204 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:20,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093500195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,205 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:20,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:20,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=150 2024-04-14T11:17:20,359 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-04-14T11:17:20,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:20,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:20,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:20,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(53): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:20,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:20,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093500396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093500397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093500397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093500406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=150 2024-04-14T11:17:20,512 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-04-14T11:17:20,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:20,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:20,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/14380760189c4ff29c33dac91f3bc2e1 2024-04-14T11:17:20,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:20,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(53): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:20,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:20,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/2e52403eed5c40ff840867d0f1e7e4fe as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2e52403eed5c40ff840867d0f1e7e4fe 2024-04-14T11:17:20,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2e52403eed5c40ff840867d0f1e7e4fe, entries=250, sequenceid=303, filesize=16.8 K 2024-04-14T11:17:20,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/db81a169aa1e4e1383103d56042ff2bd as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/db81a169aa1e4e1383103d56042ff2bd 2024-04-14T11:17:20,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/db81a169aa1e4e1383103d56042ff2bd, entries=150, sequenceid=303, filesize=12.0 K 2024-04-14T11:17:20,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/14380760189c4ff29c33dac91f3bc2e1 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/14380760189c4ff29c33dac91f3bc2e1 2024-04-14T11:17:20,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/14380760189c4ff29c33dac91f3bc2e1, entries=150, sequenceid=303, filesize=12.0 K 2024-04-14T11:17:20,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 893136536711aed76a1c4f901f8c1645 in 466ms, sequenceid=303, compaction requested=false 2024-04-14T11:17:20,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:20,665 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-04-14T11:17:20,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:20,666 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-04-14T11:17:20,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/ce722c3d0f334fb1b1207d62c8c1a10f is 50, key is test_row_0/A:col10/1713093440077/Put/seqid=0 2024-04-14T11:17:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:20,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:20,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093500735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093500736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093500741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093500742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=150 2024-04-14T11:17:20,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093500843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093500843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093500851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:20,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093500853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:21,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093501048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:21,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093501048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:21,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093501053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:21,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093501058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:21,075 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/ce722c3d0f334fb1b1207d62c8c1a10f 2024-04-14T11:17:21,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/460e7de1764c477f80795589a518f3ce is 50, key is test_row_0/B:col10/1713093440077/Put/seqid=0 2024-04-14T11:17:21,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=150 2024-04-14T11:17:21,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:21,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093501352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:21,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093501353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:21,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:21,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093501360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:21,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:21,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093501363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:21,487 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/460e7de1764c477f80795589a518f3ce 2024-04-14T11:17:21,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/c8c8bc2fee7e4b529cbbb12237aac1e1 is 50, key is test_row_0/C:col10/1713093440077/Put/seqid=0 2024-04-14T11:17:21,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093501857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:21,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:21,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093501861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:21,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093501866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:21,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093501867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:21,900 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/c8c8bc2fee7e4b529cbbb12237aac1e1 2024-04-14T11:17:21,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/ce722c3d0f334fb1b1207d62c8c1a10f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/ce722c3d0f334fb1b1207d62c8c1a10f 2024-04-14T11:17:21,911 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/ce722c3d0f334fb1b1207d62c8c1a10f, entries=150, sequenceid=317, filesize=12.0 K 2024-04-14T11:17:21,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/460e7de1764c477f80795589a518f3ce as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/460e7de1764c477f80795589a518f3ce 2024-04-14T11:17:21,917 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/460e7de1764c477f80795589a518f3ce, entries=150, sequenceid=317, filesize=12.0 K 2024-04-14T11:17:21,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/c8c8bc2fee7e4b529cbbb12237aac1e1 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/c8c8bc2fee7e4b529cbbb12237aac1e1 2024-04-14T11:17:21,923 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/c8c8bc2fee7e4b529cbbb12237aac1e1, entries=150, sequenceid=317, filesize=12.0 K 2024-04-14T11:17:21,924 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3035): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 893136536711aed76a1c4f901f8c1645 in 1258ms, sequenceid=317, compaction requested=true 2024-04-14T11:17:21,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:21,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:21,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=151 2024-04-14T11:17:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=151 2024-04-14T11:17:21,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=151, resume processing ppid=150 2024-04-14T11:17:21,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=151, ppid=150, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7210 sec 2024-04-14T11:17:21,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=150, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.7330 sec 2024-04-14T11:17:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:22,242 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:22,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-04-14T11:17:22,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/7a85f99463f24db5a08a58a2156fa509 is 50, key is test_row_0/A:col10/1713093442241/Put/seqid=0 2024-04-14T11:17:22,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093502285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=150 2024-04-14T11:17:22,310 INFO [Thread-2284 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 150 completed 2024-04-14T11:17:22,311 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-04-14T11:17:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=152 2024-04-14T11:17:22,314 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:22,315 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:22,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:22,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093502391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=152 2024-04-14T11:17:22,466 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:22,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-04-14T11:17:22,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:22,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:22,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:22,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(53): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:22,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:22,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093502594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=152 2024-04-14T11:17:22,619 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:22,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-04-14T11:17:22,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:22,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:22,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:22,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(53): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:22,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:22,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/7a85f99463f24db5a08a58a2156fa509 2024-04-14T11:17:22,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/b0268e2915574bf981abfddd322f0005 is 50, key is test_row_0/B:col10/1713093442241/Put/seqid=0 2024-04-14T11:17:22,772 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:22,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-04-14T11:17:22,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:22,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:22,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:22,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(53): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:22,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:22,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093502869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:22,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093502871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:22,880 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/bd58560fe8384777b052c75d10d73bb6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/f15d0e26dd92400b9848c8d1e6a7800b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/abbbe152e181417e87539ecf8ceaa8a2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/d6ce1b581ac141fd8296611e951f3008, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2c5aecdb2ebd41eaaadca15647edc289, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/ac16d7a2d43c48a6b08216c5cc1e38c8, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/1ce0f277edcf4a4898bf89089e0c365f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/5a0ecc302436459b8289056e6159df05, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2f61fc77ed7048f0b06463d09f487558, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/020d43d22cd044c281eca1070bf9c60f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/e278f8623c99486ea3e539c78e8d938d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/7ef43f1993d943fea215db8d234ce6d0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/a1faeb1293c54374b60ed17ace5c9d78, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/a93d3d7adb504c4ba55c1a184f37de3c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/b660bd91441c48a896894ab650602d81, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/fd4b9ee342334010a33c2afdd0911c4a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/e5918d0372944e6fb27e86d9c92555a0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/de38e29978b04e12a6457ece2adf1e6d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/cc1a6a03a37d484e98e70bead0af4877, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/686e30358dcf4eb9934550624252799d] to archive 2024-04-14T11:17:22,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093502875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:22,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093502876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:22,882 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:17:22,884 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/bd58560fe8384777b052c75d10d73bb6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/bd58560fe8384777b052c75d10d73bb6 2024-04-14T11:17:22,885 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/f15d0e26dd92400b9848c8d1e6a7800b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/f15d0e26dd92400b9848c8d1e6a7800b 2024-04-14T11:17:22,887 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/abbbe152e181417e87539ecf8ceaa8a2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/abbbe152e181417e87539ecf8ceaa8a2 2024-04-14T11:17:22,888 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/d6ce1b581ac141fd8296611e951f3008 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/d6ce1b581ac141fd8296611e951f3008 2024-04-14T11:17:22,889 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2c5aecdb2ebd41eaaadca15647edc289 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2c5aecdb2ebd41eaaadca15647edc289 2024-04-14T11:17:22,890 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/ac16d7a2d43c48a6b08216c5cc1e38c8 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/ac16d7a2d43c48a6b08216c5cc1e38c8 2024-04-14T11:17:22,892 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/1ce0f277edcf4a4898bf89089e0c365f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/1ce0f277edcf4a4898bf89089e0c365f 2024-04-14T11:17:22,893 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/5a0ecc302436459b8289056e6159df05 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/5a0ecc302436459b8289056e6159df05 2024-04-14T11:17:22,894 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2f61fc77ed7048f0b06463d09f487558 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2f61fc77ed7048f0b06463d09f487558 2024-04-14T11:17:22,895 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/020d43d22cd044c281eca1070bf9c60f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/020d43d22cd044c281eca1070bf9c60f 2024-04-14T11:17:22,896 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/e278f8623c99486ea3e539c78e8d938d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/e278f8623c99486ea3e539c78e8d938d 2024-04-14T11:17:22,898 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/7ef43f1993d943fea215db8d234ce6d0 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/7ef43f1993d943fea215db8d234ce6d0 2024-04-14T11:17:22,899 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/a1faeb1293c54374b60ed17ace5c9d78 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/a1faeb1293c54374b60ed17ace5c9d78 2024-04-14T11:17:22,902 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/a93d3d7adb504c4ba55c1a184f37de3c to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/a93d3d7adb504c4ba55c1a184f37de3c 2024-04-14T11:17:22,904 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/b660bd91441c48a896894ab650602d81 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/b660bd91441c48a896894ab650602d81 2024-04-14T11:17:22,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:22,905 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/fd4b9ee342334010a33c2afdd0911c4a to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/fd4b9ee342334010a33c2afdd0911c4a 2024-04-14T11:17:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093502898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:22,906 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/e5918d0372944e6fb27e86d9c92555a0 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/e5918d0372944e6fb27e86d9c92555a0 2024-04-14T11:17:22,908 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/de38e29978b04e12a6457ece2adf1e6d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/de38e29978b04e12a6457ece2adf1e6d 2024-04-14T11:17:22,909 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/cc1a6a03a37d484e98e70bead0af4877 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/cc1a6a03a37d484e98e70bead0af4877 2024-04-14T11:17:22,910 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/686e30358dcf4eb9934550624252799d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/686e30358dcf4eb9934550624252799d 2024-04-14T11:17:22,916 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/939bf8d4a8e347a3a23c7e6e888382a9, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/f271f3503bd149b0a1aa0231df6f75c6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/13c602a016bf4fc7aa7e8f3b609bfee0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/31a6162cd55747509c7b99eb8a9f909e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/cc275a271ab0495a8166ea57ded680ce, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/e469df02f3974627a24b0296019f0da2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7324e1bee8494aeeac4ce724f150ced2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7dee022506ae4f21a932f25cb584f69f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/5db2e1cbe333416d8986c4c9f8bd18e5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b838039d794a4844b49799d8d03410ee, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/dc49ec4b4ffb4c27a6f0ce9df98401a6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7773d08fe416477e8d46c10c0c4d5665, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/929d423bb7214f6d8e60458248b7286d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/4ed93992f061433298ae43e3008c84bf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/0f9ad6055d4d48caa08a358d4666b954, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/c24dc2ebbc1c4664bc91f8908dd332b3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/a838fc6c87be441483f9ed601ceb0525, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/c970a67fa6664bf393938b751e58be72, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/0b3ef9c50f4045a590b0506529207e31, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/2109eed54f904a8c8991af8f34325193] to archive 2024-04-14T11:17:22,916 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:17:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=152 2024-04-14T11:17:22,918 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/939bf8d4a8e347a3a23c7e6e888382a9 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/939bf8d4a8e347a3a23c7e6e888382a9 2024-04-14T11:17:22,919 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/f271f3503bd149b0a1aa0231df6f75c6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/f271f3503bd149b0a1aa0231df6f75c6 2024-04-14T11:17:22,920 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/13c602a016bf4fc7aa7e8f3b609bfee0 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/13c602a016bf4fc7aa7e8f3b609bfee0 2024-04-14T11:17:22,922 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/31a6162cd55747509c7b99eb8a9f909e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/31a6162cd55747509c7b99eb8a9f909e 2024-04-14T11:17:22,923 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/cc275a271ab0495a8166ea57ded680ce to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/cc275a271ab0495a8166ea57ded680ce 2024-04-14T11:17:22,924 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/e469df02f3974627a24b0296019f0da2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/e469df02f3974627a24b0296019f0da2 2024-04-14T11:17:22,925 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:22,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-04-14T11:17:22,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:22,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:22,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:22,926 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7324e1bee8494aeeac4ce724f150ced2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7324e1bee8494aeeac4ce724f150ced2 2024-04-14T11:17:22,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(53): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:22,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:22,929 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7dee022506ae4f21a932f25cb584f69f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7dee022506ae4f21a932f25cb584f69f 2024-04-14T11:17:22,930 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/5db2e1cbe333416d8986c4c9f8bd18e5 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/5db2e1cbe333416d8986c4c9f8bd18e5 2024-04-14T11:17:22,934 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b838039d794a4844b49799d8d03410ee to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b838039d794a4844b49799d8d03410ee 2024-04-14T11:17:22,935 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/dc49ec4b4ffb4c27a6f0ce9df98401a6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/dc49ec4b4ffb4c27a6f0ce9df98401a6 2024-04-14T11:17:22,936 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7773d08fe416477e8d46c10c0c4d5665 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7773d08fe416477e8d46c10c0c4d5665 2024-04-14T11:17:22,938 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/929d423bb7214f6d8e60458248b7286d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/929d423bb7214f6d8e60458248b7286d 2024-04-14T11:17:22,940 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/4ed93992f061433298ae43e3008c84bf to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/4ed93992f061433298ae43e3008c84bf 2024-04-14T11:17:22,942 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/0f9ad6055d4d48caa08a358d4666b954 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/0f9ad6055d4d48caa08a358d4666b954 2024-04-14T11:17:22,943 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/c24dc2ebbc1c4664bc91f8908dd332b3 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/c24dc2ebbc1c4664bc91f8908dd332b3 2024-04-14T11:17:22,945 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/a838fc6c87be441483f9ed601ceb0525 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/a838fc6c87be441483f9ed601ceb0525 2024-04-14T11:17:22,947 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/c970a67fa6664bf393938b751e58be72 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/c970a67fa6664bf393938b751e58be72 2024-04-14T11:17:22,948 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/0b3ef9c50f4045a590b0506529207e31 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/0b3ef9c50f4045a590b0506529207e31 2024-04-14T11:17:22,950 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/2109eed54f904a8c8991af8f34325193 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/2109eed54f904a8c8991af8f34325193 2024-04-14T11:17:22,952 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/69e4b31a0b2a4e75bddaf53cc5263412, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/90ee543f1dab4f9a98372dbfe73d1f2a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/b7975f3e707c44e1a6667ca2ccd785f5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/f8a7ffb1b58e485eb1eccd72d121ab3e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/737a5ed110644ebfb39749d5a3362aa2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/aaa5954f44fd49139b5215c906939e38, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/e9a52fcb3dbe46d7ac433bf8deed11c5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/c26d2ee7507d43e3879909af0748b8c4, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a697061b474141dcbaba423a44764180, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a0b9a18969aa4015b47538d9be5123a3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/facfcb2cd4c645bf9a982a61740a2228, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/0b1ddf82ac0c4737bf920a990a1c81e5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/f24272593a654670ad088bece4da203f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/836380589c254291917c8b5a2b90caea, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/e0fa6f68445e467da5fc844a9e4def02, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/5c0c5734db02480fb43ca9296b8002fe, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/778bdeaae8c3448c8d65dd914fdf5311, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/04d4a39ae219405ebe7930d809130301, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/72253e6f5ab94ba1a0563e911d9897cf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/5c05ccef6c5043c4992b8510f6b78e13] to archive 2024-04-14T11:17:22,953 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:17:22,955 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/69e4b31a0b2a4e75bddaf53cc5263412 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/69e4b31a0b2a4e75bddaf53cc5263412 2024-04-14T11:17:22,957 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/90ee543f1dab4f9a98372dbfe73d1f2a to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/90ee543f1dab4f9a98372dbfe73d1f2a 2024-04-14T11:17:22,958 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/b7975f3e707c44e1a6667ca2ccd785f5 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/b7975f3e707c44e1a6667ca2ccd785f5 2024-04-14T11:17:22,960 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/f8a7ffb1b58e485eb1eccd72d121ab3e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/f8a7ffb1b58e485eb1eccd72d121ab3e 2024-04-14T11:17:22,963 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/737a5ed110644ebfb39749d5a3362aa2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/737a5ed110644ebfb39749d5a3362aa2 2024-04-14T11:17:22,964 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/aaa5954f44fd49139b5215c906939e38 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/aaa5954f44fd49139b5215c906939e38 2024-04-14T11:17:22,965 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/e9a52fcb3dbe46d7ac433bf8deed11c5 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/e9a52fcb3dbe46d7ac433bf8deed11c5 2024-04-14T11:17:22,967 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/c26d2ee7507d43e3879909af0748b8c4 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/c26d2ee7507d43e3879909af0748b8c4 2024-04-14T11:17:22,968 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a697061b474141dcbaba423a44764180 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a697061b474141dcbaba423a44764180 2024-04-14T11:17:22,969 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a0b9a18969aa4015b47538d9be5123a3 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a0b9a18969aa4015b47538d9be5123a3 2024-04-14T11:17:22,971 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/facfcb2cd4c645bf9a982a61740a2228 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/facfcb2cd4c645bf9a982a61740a2228 2024-04-14T11:17:22,972 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/0b1ddf82ac0c4737bf920a990a1c81e5 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/0b1ddf82ac0c4737bf920a990a1c81e5 2024-04-14T11:17:22,973 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/f24272593a654670ad088bece4da203f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/f24272593a654670ad088bece4da203f 2024-04-14T11:17:22,975 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/836380589c254291917c8b5a2b90caea to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/836380589c254291917c8b5a2b90caea 2024-04-14T11:17:22,976 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/e0fa6f68445e467da5fc844a9e4def02 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/e0fa6f68445e467da5fc844a9e4def02 2024-04-14T11:17:22,977 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/5c0c5734db02480fb43ca9296b8002fe to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/5c0c5734db02480fb43ca9296b8002fe 2024-04-14T11:17:22,978 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/778bdeaae8c3448c8d65dd914fdf5311 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/778bdeaae8c3448c8d65dd914fdf5311 2024-04-14T11:17:22,980 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/04d4a39ae219405ebe7930d809130301 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/04d4a39ae219405ebe7930d809130301 2024-04-14T11:17:22,982 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/72253e6f5ab94ba1a0563e911d9897cf to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/72253e6f5ab94ba1a0563e911d9897cf 2024-04-14T11:17:22,983 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/5c05ccef6c5043c4992b8510f6b78e13 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/5c05ccef6c5043c4992b8510f6b78e13 2024-04-14T11:17:23,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/b0268e2915574bf981abfddd322f0005 2024-04-14T11:17:23,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/8c8f9305f7894c8a9ad934ee85a45565 is 50, key is test_row_0/C:col10/1713093442241/Put/seqid=0 2024-04-14T11:17:23,078 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:23,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-04-14T11:17:23,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:23,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:23,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:23,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(53): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:23,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:23,231 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:23,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-04-14T11:17:23,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:23,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:23,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:23,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(53): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:23,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:23,384 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:23,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-04-14T11:17:23,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:23,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:23,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:23,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(53): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:23,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:23,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:23,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093503410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=152 2024-04-14T11:17:23,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/8c8f9305f7894c8a9ad934ee85a45565 2024-04-14T11:17:23,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/7a85f99463f24db5a08a58a2156fa509 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/7a85f99463f24db5a08a58a2156fa509 2024-04-14T11:17:23,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/7a85f99463f24db5a08a58a2156fa509, entries=200, sequenceid=341, filesize=14.4 K 2024-04-14T11:17:23,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/b0268e2915574bf981abfddd322f0005 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b0268e2915574bf981abfddd322f0005 2024-04-14T11:17:23,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b0268e2915574bf981abfddd322f0005, entries=150, sequenceid=341, filesize=12.0 K 2024-04-14T11:17:23,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/8c8f9305f7894c8a9ad934ee85a45565 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/8c8f9305f7894c8a9ad934ee85a45565 2024-04-14T11:17:23,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/8c8f9305f7894c8a9ad934ee85a45565, entries=150, sequenceid=341, filesize=12.0 K 2024-04-14T11:17:23,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 893136536711aed76a1c4f901f8c1645 in 1277ms, sequenceid=341, compaction requested=true 2024-04-14T11:17:23,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:23,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:23,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:23,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:23,519 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:17:23,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:23,519 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:17:23,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:23,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:23,520 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57138 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:17:23,521 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/A is initiating minor compaction (all files) 2024-04-14T11:17:23,521 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/A in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:23,521 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2d3f47b9c8bd484cb4a5c1bd79b00e45, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2e52403eed5c40ff840867d0f1e7e4fe, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/ce722c3d0f334fb1b1207d62c8c1a10f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/7a85f99463f24db5a08a58a2156fa509] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=55.8 K 2024-04-14T11:17:23,521 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:17:23,521 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/B is initiating minor compaction (all files) 2024-04-14T11:17:23,521 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/B in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:23,521 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/bd95ff9ceb6148c5a93338dff5c2a3ec, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/db81a169aa1e4e1383103d56042ff2bd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/460e7de1764c477f80795589a518f3ce, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b0268e2915574bf981abfddd322f0005] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=48.7 K 2024-04-14T11:17:23,521 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2d3f47b9c8bd484cb4a5c1bd79b00e45, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1713093439606 2024-04-14T11:17:23,521 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting bd95ff9ceb6148c5a93338dff5c2a3ec, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1713093439606 2024-04-14T11:17:23,522 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2e52403eed5c40ff840867d0f1e7e4fe, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1713093439943 2024-04-14T11:17:23,522 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting db81a169aa1e4e1383103d56042ff2bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1713093439943 2024-04-14T11:17:23,522 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting ce722c3d0f334fb1b1207d62c8c1a10f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1713093440077 2024-04-14T11:17:23,522 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 460e7de1764c477f80795589a518f3ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1713093440077 2024-04-14T11:17:23,522 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7a85f99463f24db5a08a58a2156fa509, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1713093440725 2024-04-14T11:17:23,523 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting b0268e2915574bf981abfddd322f0005, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1713093440740 2024-04-14T11:17:23,531 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#B#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:23,532 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#A#compaction#508 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:23,532 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/1f94b361502f4d5fb6d58f55932102cf is 50, key is test_row_0/B:col10/1713093442241/Put/seqid=0 2024-04-14T11:17:23,532 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/bc9b3e67717249bc9cfa9f38e481e56d is 50, key is test_row_0/A:col10/1713093442241/Put/seqid=0 2024-04-14T11:17:23,537 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:23,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-04-14T11:17:23,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:23,538 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-04-14T11:17:23,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/72a57a6f83f14b55bd271b873854df3e is 50, key is test_row_0/A:col10/1713093442284/Put/seqid=0 2024-04-14T11:17:23,546 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/bc9b3e67717249bc9cfa9f38e481e56d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/bc9b3e67717249bc9cfa9f38e481e56d 2024-04-14T11:17:23,553 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 893136536711aed76a1c4f901f8c1645/A of 893136536711aed76a1c4f901f8c1645 into bc9b3e67717249bc9cfa9f38e481e56d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:23,553 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:23,553 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/A, priority=12, startTime=1713093443519; duration=0sec 2024-04-14T11:17:23,553 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:23,553 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:A 2024-04-14T11:17:23,553 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:17:23,555 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/72a57a6f83f14b55bd271b873854df3e 2024-04-14T11:17:23,555 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:17:23,555 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/C is initiating minor compaction (all files) 2024-04-14T11:17:23,555 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/C in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:23,555 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/8d29d736fbe24798bd6f45882f203370, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/14380760189c4ff29c33dac91f3bc2e1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/c8c8bc2fee7e4b529cbbb12237aac1e1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/8c8f9305f7894c8a9ad934ee85a45565] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=48.7 K 2024-04-14T11:17:23,556 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8d29d736fbe24798bd6f45882f203370, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1713093439606 2024-04-14T11:17:23,556 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 14380760189c4ff29c33dac91f3bc2e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1713093439943 2024-04-14T11:17:23,557 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c8c8bc2fee7e4b529cbbb12237aac1e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1713093440077 2024-04-14T11:17:23,557 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8c8f9305f7894c8a9ad934ee85a45565, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1713093440740 2024-04-14T11:17:23,566 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#C#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:23,567 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/8112e1f8abb449c5afa67564607d8c39 is 50, key is test_row_0/C:col10/1713093442241/Put/seqid=0 2024-04-14T11:17:23,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/ebf5050c956641529112cc35819a7908 is 50, key is test_row_0/B:col10/1713093442284/Put/seqid=0 2024-04-14T11:17:23,576 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/ebf5050c956641529112cc35819a7908 2024-04-14T11:17:23,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/fbd30c21b91947d790e41a78a072eead is 50, key is test_row_0/C:col10/1713093442284/Put/seqid=0 2024-04-14T11:17:23,943 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/1f94b361502f4d5fb6d58f55932102cf as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/1f94b361502f4d5fb6d58f55932102cf 2024-04-14T11:17:23,948 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 893136536711aed76a1c4f901f8c1645/B of 893136536711aed76a1c4f901f8c1645 into 1f94b361502f4d5fb6d58f55932102cf(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:23,949 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:23,949 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/B, priority=12, startTime=1713093443519; duration=0sec 2024-04-14T11:17:23,949 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:23,949 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:B 2024-04-14T11:17:23,978 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/8112e1f8abb449c5afa67564607d8c39 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/8112e1f8abb449c5afa67564607d8c39 2024-04-14T11:17:23,984 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in 893136536711aed76a1c4f901f8c1645/C of 893136536711aed76a1c4f901f8c1645 into 8112e1f8abb449c5afa67564607d8c39(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:23,984 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:23,984 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/C, priority=12, startTime=1713093443519; duration=0sec 2024-04-14T11:17:23,984 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:23,984 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:C 2024-04-14T11:17:23,992 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/fbd30c21b91947d790e41a78a072eead 2024-04-14T11:17:23,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/72a57a6f83f14b55bd271b873854df3e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/72a57a6f83f14b55bd271b873854df3e 2024-04-14T11:17:24,006 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/72a57a6f83f14b55bd271b873854df3e, entries=150, sequenceid=353, filesize=12.0 K 2024-04-14T11:17:24,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/ebf5050c956641529112cc35819a7908 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/ebf5050c956641529112cc35819a7908 2024-04-14T11:17:24,013 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/ebf5050c956641529112cc35819a7908, entries=150, sequenceid=353, filesize=12.0 K 2024-04-14T11:17:24,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/fbd30c21b91947d790e41a78a072eead as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/fbd30c21b91947d790e41a78a072eead 2024-04-14T11:17:24,026 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/fbd30c21b91947d790e41a78a072eead, entries=150, sequenceid=353, filesize=12.0 K 2024-04-14T11:17:24,027 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3035): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 893136536711aed76a1c4f901f8c1645 in 489ms, sequenceid=353, compaction requested=false 2024-04-14T11:17:24,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:24,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:24,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=153 2024-04-14T11:17:24,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=153 2024-04-14T11:17:24,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=153, resume processing ppid=152 2024-04-14T11:17:24,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=153, ppid=152, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7130 sec 2024-04-14T11:17:24,032 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=152, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.7200 sec 2024-04-14T11:17:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=152 2024-04-14T11:17:24,419 INFO [Thread-2284 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 152 completed 2024-04-14T11:17:24,422 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-04-14T11:17:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=154 2024-04-14T11:17:24,425 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:24,426 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(88): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:24,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:24,442 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:24,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-04-14T11:17:24,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/93db23979b9c48ce8b13498207e8ebba is 50, key is test_row_0/A:col10/1713093444436/Put/seqid=0 2024-04-14T11:17:24,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/93db23979b9c48ce8b13498207e8ebba 2024-04-14T11:17:24,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/85efeb386e77495c9957f164ec03320d is 50, key is test_row_0/B:col10/1713093444436/Put/seqid=0 2024-04-14T11:17:24,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/85efeb386e77495c9957f164ec03320d 2024-04-14T11:17:24,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/a7d7a55b080b4a2f8e28ebafadcb0ada is 50, key is test_row_0/C:col10/1713093444436/Put/seqid=0 2024-04-14T11:17:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=154 2024-04-14T11:17:24,578 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:24,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-04-14T11:17:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:24,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(53): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:24,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093504577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:24,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093504683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=154 2024-04-14T11:17:24,731 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:24,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-04-14T11:17:24,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:24,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:24,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:24,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(53): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:24,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:24,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093504881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:24,885 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:24,885 DEBUG [Thread-2276 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4147 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., hostname=jenkins-hbase6.apache.org,33239,1713093321027, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2024-04-14T11:17:24,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-04-14T11:17:24,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:24,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:24,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:24,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(53): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:24,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:24,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093504887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:24,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093504892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:24,897 DEBUG [Thread-2282 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4154 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., hostname=jenkins-hbase6.apache.org,33239,1713093321027, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2024-04-14T11:17:24,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093504893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:24,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093504894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:24,900 DEBUG [Thread-2274 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4164 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., hostname=jenkins-hbase6.apache.org,33239,1713093321027, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2024-04-14T11:17:24,901 DEBUG [Thread-2278 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4158 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., hostname=jenkins-hbase6.apache.org,33239,1713093321027, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2024-04-14T11:17:24,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/a7d7a55b080b4a2f8e28ebafadcb0ada 2024-04-14T11:17:24,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/93db23979b9c48ce8b13498207e8ebba as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/93db23979b9c48ce8b13498207e8ebba 2024-04-14T11:17:24,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/93db23979b9c48ce8b13498207e8ebba, entries=200, sequenceid=367, filesize=14.4 K 2024-04-14T11:17:24,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/85efeb386e77495c9957f164ec03320d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/85efeb386e77495c9957f164ec03320d 2024-04-14T11:17:24,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/85efeb386e77495c9957f164ec03320d, entries=150, sequenceid=367, filesize=12.0 K 2024-04-14T11:17:24,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/a7d7a55b080b4a2f8e28ebafadcb0ada as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a7d7a55b080b4a2f8e28ebafadcb0ada 2024-04-14T11:17:24,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a7d7a55b080b4a2f8e28ebafadcb0ada, entries=150, sequenceid=367, filesize=12.0 K 2024-04-14T11:17:24,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 893136536711aed76a1c4f901f8c1645 in 481ms, sequenceid=367, compaction requested=true 2024-04-14T11:17:24,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:24,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:24,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:24,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:24,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:24,923 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:24,923 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:24,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:24,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:24,925 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39430 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:24,925 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/A is initiating minor compaction (all files) 2024-04-14T11:17:24,925 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/A in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:24,925 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/bc9b3e67717249bc9cfa9f38e481e56d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/72a57a6f83f14b55bd271b873854df3e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/93db23979b9c48ce8b13498207e8ebba] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=38.5 K 2024-04-14T11:17:24,925 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36990 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:24,925 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/B is initiating minor compaction (all files) 2024-04-14T11:17:24,925 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/B in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:24,925 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/1f94b361502f4d5fb6d58f55932102cf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/ebf5050c956641529112cc35819a7908, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/85efeb386e77495c9957f164ec03320d] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=36.1 K 2024-04-14T11:17:24,930 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting bc9b3e67717249bc9cfa9f38e481e56d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1713093440740 2024-04-14T11:17:24,932 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 1f94b361502f4d5fb6d58f55932102cf, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1713093440740 2024-04-14T11:17:24,932 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 72a57a6f83f14b55bd271b873854df3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1713093442274 2024-04-14T11:17:24,932 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting ebf5050c956641529112cc35819a7908, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1713093442274 2024-04-14T11:17:24,932 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 93db23979b9c48ce8b13498207e8ebba, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1713093444418 2024-04-14T11:17:24,933 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 85efeb386e77495c9957f164ec03320d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1713093444427 2024-04-14T11:17:24,939 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#A#compaction#517 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:24,940 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#B#compaction#518 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:24,940 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/32e48cb8270940018dd41f00e63e6fab is 50, key is test_row_0/A:col10/1713093444436/Put/seqid=0 2024-04-14T11:17:24,940 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/141acf68b9fc4f7481678ea54aeab117 is 50, key is test_row_0/B:col10/1713093444436/Put/seqid=0 2024-04-14T11:17:24,951 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/32e48cb8270940018dd41f00e63e6fab as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/32e48cb8270940018dd41f00e63e6fab 2024-04-14T11:17:24,952 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/141acf68b9fc4f7481678ea54aeab117 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/141acf68b9fc4f7481678ea54aeab117 2024-04-14T11:17:24,958 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/A of 893136536711aed76a1c4f901f8c1645 into 32e48cb8270940018dd41f00e63e6fab(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:24,959 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:24,959 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/A, priority=13, startTime=1713093444923; duration=0sec 2024-04-14T11:17:24,959 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:24,959 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:A 2024-04-14T11:17:24,959 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:24,959 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/B of 893136536711aed76a1c4f901f8c1645 into 141acf68b9fc4f7481678ea54aeab117(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:24,959 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:24,959 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/B, priority=13, startTime=1713093444923; duration=0sec 2024-04-14T11:17:24,959 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:24,959 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:B 2024-04-14T11:17:24,960 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36990 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:24,960 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/C is initiating minor compaction (all files) 2024-04-14T11:17:24,960 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/C in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:24,960 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/8112e1f8abb449c5afa67564607d8c39, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/fbd30c21b91947d790e41a78a072eead, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a7d7a55b080b4a2f8e28ebafadcb0ada] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=36.1 K 2024-04-14T11:17:24,960 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 8112e1f8abb449c5afa67564607d8c39, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1713093440740 2024-04-14T11:17:24,961 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting fbd30c21b91947d790e41a78a072eead, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1713093442274 2024-04-14T11:17:24,961 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting a7d7a55b080b4a2f8e28ebafadcb0ada, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1713093444427 2024-04-14T11:17:24,967 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#C#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:24,967 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/1f32bcb4eafb4e69853bff88638eea3d is 50, key is test_row_0/C:col10/1713093444436/Put/seqid=0 2024-04-14T11:17:24,996 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/1f32bcb4eafb4e69853bff88638eea3d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/1f32bcb4eafb4e69853bff88638eea3d 2024-04-14T11:17:25,002 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/C of 893136536711aed76a1c4f901f8c1645 into 1f32bcb4eafb4e69853bff88638eea3d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:25,002 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:25,002 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/C, priority=13, startTime=1713093444923; duration=0sec 2024-04-14T11:17:25,002 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:25,003 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:C 2024-04-14T11:17:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=154 2024-04-14T11:17:25,038 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:25,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-04-14T11:17:25,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:25,039 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-04-14T11:17:25,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/b997b725aa984e11a823fcc125f962f6 is 50, key is test_row_0/A:col10/1713093444576/Put/seqid=0 2024-04-14T11:17:25,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:25,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:25,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093505222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:25,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:25,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093505327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:25,460 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/b997b725aa984e11a823fcc125f962f6 2024-04-14T11:17:25,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/b5c7a5dc0cc54962b06df0dbab9230c7 is 50, key is test_row_0/B:col10/1713093444576/Put/seqid=0 2024-04-14T11:17:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=154 2024-04-14T11:17:25,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093505533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:25,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:25,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093505840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:25,873 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/b5c7a5dc0cc54962b06df0dbab9230c7 2024-04-14T11:17:25,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/a098e9ba685c4cdeb36cf242e79f7592 is 50, key is test_row_0/C:col10/1713093444576/Put/seqid=0 2024-04-14T11:17:25,886 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/a098e9ba685c4cdeb36cf242e79f7592 2024-04-14T11:17:25,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/b997b725aa984e11a823fcc125f962f6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/b997b725aa984e11a823fcc125f962f6 2024-04-14T11:17:25,897 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/b997b725aa984e11a823fcc125f962f6, entries=150, sequenceid=395, filesize=12.0 K 2024-04-14T11:17:25,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/b5c7a5dc0cc54962b06df0dbab9230c7 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b5c7a5dc0cc54962b06df0dbab9230c7 2024-04-14T11:17:25,905 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b5c7a5dc0cc54962b06df0dbab9230c7, entries=150, sequenceid=395, filesize=12.0 K 2024-04-14T11:17:25,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/a098e9ba685c4cdeb36cf242e79f7592 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a098e9ba685c4cdeb36cf242e79f7592 2024-04-14T11:17:25,911 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a098e9ba685c4cdeb36cf242e79f7592, entries=150, sequenceid=395, filesize=12.0 K 2024-04-14T11:17:25,912 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3035): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 893136536711aed76a1c4f901f8c1645 in 873ms, sequenceid=395, compaction requested=false 2024-04-14T11:17:25,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:25,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:25,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=155 2024-04-14T11:17:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=155 2024-04-14T11:17:25,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=155, resume processing ppid=154 2024-04-14T11:17:25,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=155, ppid=154, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4870 sec 2024-04-14T11:17:25,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=154, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4940 sec 2024-04-14T11:17:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:26,348 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:26,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-04-14T11:17:26,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/88d630e6719b45a99bb163f969c4fe70 is 50, key is test_row_0/A:col10/1713093445221/Put/seqid=0 2024-04-14T11:17:26,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/88d630e6719b45a99bb163f969c4fe70 2024-04-14T11:17:26,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/91ad8b999cc34ea29908addd2ee85979 is 50, key is test_row_0/B:col10/1713093445221/Put/seqid=0 2024-04-14T11:17:26,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093506458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=154 2024-04-14T11:17:26,530 INFO [Thread-2284 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 154 completed 2024-04-14T11:17:26,532 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-04-14T11:17:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=156 2024-04-14T11:17:26,535 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:26,536 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:26,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:26,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093506563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=156 2024-04-14T11:17:26,687 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:26,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-04-14T11:17:26,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:26,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:26,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:26,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(53): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:26,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:26,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093506768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:26,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/91ad8b999cc34ea29908addd2ee85979 2024-04-14T11:17:26,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/75d9b5c4b86b4deeb407c9d07c4cdc37 is 50, key is test_row_0/C:col10/1713093445221/Put/seqid=0 2024-04-14T11:17:26,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/75d9b5c4b86b4deeb407c9d07c4cdc37 2024-04-14T11:17:26,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/88d630e6719b45a99bb163f969c4fe70 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/88d630e6719b45a99bb163f969c4fe70 2024-04-14T11:17:26,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/88d630e6719b45a99bb163f969c4fe70, entries=150, sequenceid=407, filesize=12.0 K 2024-04-14T11:17:26,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/91ad8b999cc34ea29908addd2ee85979 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/91ad8b999cc34ea29908addd2ee85979 2024-04-14T11:17:26,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/91ad8b999cc34ea29908addd2ee85979, entries=150, sequenceid=407, filesize=12.0 K 2024-04-14T11:17:26,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/75d9b5c4b86b4deeb407c9d07c4cdc37 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/75d9b5c4b86b4deeb407c9d07c4cdc37 2024-04-14T11:17:26,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/75d9b5c4b86b4deeb407c9d07c4cdc37, entries=150, sequenceid=407, filesize=12.0 K 2024-04-14T11:17:26,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 893136536711aed76a1c4f901f8c1645 in 466ms, sequenceid=407, compaction requested=true 2024-04-14T11:17:26,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:26,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:26,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:26,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:26,814 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:26,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:26,814 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:26,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:26,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:26,815 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37092 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:26,815 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/A is initiating minor compaction (all files) 2024-04-14T11:17:26,815 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/A in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:26,815 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37092 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:26,815 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/B is initiating minor compaction (all files) 2024-04-14T11:17:26,815 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/32e48cb8270940018dd41f00e63e6fab, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/b997b725aa984e11a823fcc125f962f6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/88d630e6719b45a99bb163f969c4fe70] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=36.2 K 2024-04-14T11:17:26,815 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/B in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:26,815 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/141acf68b9fc4f7481678ea54aeab117, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b5c7a5dc0cc54962b06df0dbab9230c7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/91ad8b999cc34ea29908addd2ee85979] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=36.2 K 2024-04-14T11:17:26,816 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 32e48cb8270940018dd41f00e63e6fab, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1713093444427 2024-04-14T11:17:26,816 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 141acf68b9fc4f7481678ea54aeab117, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1713093444427 2024-04-14T11:17:26,816 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting b997b725aa984e11a823fcc125f962f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1713093444554 2024-04-14T11:17:26,816 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting b5c7a5dc0cc54962b06df0dbab9230c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1713093444554 2024-04-14T11:17:26,816 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 88d630e6719b45a99bb163f969c4fe70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1713093445199 2024-04-14T11:17:26,817 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 91ad8b999cc34ea29908addd2ee85979, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1713093445199 2024-04-14T11:17:26,824 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#B#compaction#527 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:26,825 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#A#compaction#526 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:26,825 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/eb2f22e471b546279cd4dc6b5eb5eb24 is 50, key is test_row_0/B:col10/1713093445221/Put/seqid=0 2024-04-14T11:17:26,825 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/78102fdd10b74c519fda5efc489a55e6 is 50, key is test_row_0/A:col10/1713093445221/Put/seqid=0 2024-04-14T11:17:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=156 2024-04-14T11:17:26,839 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/eb2f22e471b546279cd4dc6b5eb5eb24 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/eb2f22e471b546279cd4dc6b5eb5eb24 2024-04-14T11:17:26,840 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:26,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-04-14T11:17:26,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:26,841 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-04-14T11:17:26,845 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/B of 893136536711aed76a1c4f901f8c1645 into eb2f22e471b546279cd4dc6b5eb5eb24(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:26,845 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:26,845 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/B, priority=13, startTime=1713093446814; duration=0sec 2024-04-14T11:17:26,845 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:26,845 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:B 2024-04-14T11:17:26,845 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:26,848 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37092 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:26,848 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/C is initiating minor compaction (all files) 2024-04-14T11:17:26,848 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/C in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:26,848 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/1f32bcb4eafb4e69853bff88638eea3d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a098e9ba685c4cdeb36cf242e79f7592, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/75d9b5c4b86b4deeb407c9d07c4cdc37] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=36.2 K 2024-04-14T11:17:26,848 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 1f32bcb4eafb4e69853bff88638eea3d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1713093444427 2024-04-14T11:17:26,849 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting a098e9ba685c4cdeb36cf242e79f7592, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1713093444554 2024-04-14T11:17:26,850 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 75d9b5c4b86b4deeb407c9d07c4cdc37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1713093445199 2024-04-14T11:17:26,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/43596f20781b40f894a48d3cdc3fb898 is 50, key is test_row_0/A:col10/1713093446441/Put/seqid=0 2024-04-14T11:17:26,855 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/43596f20781b40f894a48d3cdc3fb898 2024-04-14T11:17:26,857 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#C#compaction#529 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:26,857 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/b14d157056234880a23448f6612177ec is 50, key is test_row_0/C:col10/1713093445221/Put/seqid=0 2024-04-14T11:17:26,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/623fe7b1ff1949719425cf51a3f13939 is 50, key is test_row_0/B:col10/1713093446441/Put/seqid=0 2024-04-14T11:17:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:27,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:27,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093507122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=156 2024-04-14T11:17:27,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093507228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:27,241 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/78102fdd10b74c519fda5efc489a55e6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/78102fdd10b74c519fda5efc489a55e6 2024-04-14T11:17:27,247 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/A of 893136536711aed76a1c4f901f8c1645 into 78102fdd10b74c519fda5efc489a55e6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:27,247 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:27,247 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/A, priority=13, startTime=1713093446814; duration=0sec 2024-04-14T11:17:27,247 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:27,247 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:A 2024-04-14T11:17:27,268 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/623fe7b1ff1949719425cf51a3f13939 2024-04-14T11:17:27,269 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/b14d157056234880a23448f6612177ec as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/b14d157056234880a23448f6612177ec 2024-04-14T11:17:27,275 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/C of 893136536711aed76a1c4f901f8c1645 into b14d157056234880a23448f6612177ec(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:27,276 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:27,276 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/C, priority=13, startTime=1713093446814; duration=0sec 2024-04-14T11:17:27,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/2b30c86af373414db5c2ce583d5f6161 is 50, key is test_row_0/C:col10/1713093446441/Put/seqid=0 2024-04-14T11:17:27,278 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:27,278 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:C 2024-04-14T11:17:27,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:27,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093507433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=156 2024-04-14T11:17:27,685 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/2b30c86af373414db5c2ce583d5f6161 2024-04-14T11:17:27,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/43596f20781b40f894a48d3cdc3fb898 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/43596f20781b40f894a48d3cdc3fb898 2024-04-14T11:17:27,696 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/43596f20781b40f894a48d3cdc3fb898, entries=150, sequenceid=431, filesize=12.0 K 2024-04-14T11:17:27,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/623fe7b1ff1949719425cf51a3f13939 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/623fe7b1ff1949719425cf51a3f13939 2024-04-14T11:17:27,702 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/623fe7b1ff1949719425cf51a3f13939, entries=150, sequenceid=431, filesize=12.0 K 2024-04-14T11:17:27,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/2b30c86af373414db5c2ce583d5f6161 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/2b30c86af373414db5c2ce583d5f6161 2024-04-14T11:17:27,708 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/2b30c86af373414db5c2ce583d5f6161, entries=150, sequenceid=431, filesize=12.0 K 2024-04-14T11:17:27,709 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3035): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 893136536711aed76a1c4f901f8c1645 in 868ms, sequenceid=431, compaction requested=false 2024-04-14T11:17:27,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:27,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:27,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=157 2024-04-14T11:17:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=157 2024-04-14T11:17:27,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=157, resume processing ppid=156 2024-04-14T11:17:27,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=157, ppid=156, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1740 sec 2024-04-14T11:17:27,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=156, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.1800 sec 2024-04-14T11:17:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:27,738 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:27,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-04-14T11:17:27,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/28c09d6567314c07aa5af3128c77fbc8 is 50, key is test_row_0/A:col10/1713093447114/Put/seqid=0 2024-04-14T11:17:27,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093507844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:27,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093507950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:28,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/28c09d6567314c07aa5af3128c77fbc8 2024-04-14T11:17:28,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093508154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:28,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/7466a5145dee483ea7126a9d51982086 is 50, key is test_row_0/B:col10/1713093447114/Put/seqid=0 2024-04-14T11:17:28,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/7466a5145dee483ea7126a9d51982086 2024-04-14T11:17:28,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/9d999046b1304d159d6265cc513e5531 is 50, key is test_row_0/C:col10/1713093447114/Put/seqid=0 2024-04-14T11:17:28,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/9d999046b1304d159d6265cc513e5531 2024-04-14T11:17:28,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/28c09d6567314c07aa5af3128c77fbc8 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/28c09d6567314c07aa5af3128c77fbc8 2024-04-14T11:17:28,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/28c09d6567314c07aa5af3128c77fbc8, entries=200, sequenceid=447, filesize=14.4 K 2024-04-14T11:17:28,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/7466a5145dee483ea7126a9d51982086 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7466a5145dee483ea7126a9d51982086 2024-04-14T11:17:28,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7466a5145dee483ea7126a9d51982086, entries=150, sequenceid=447, filesize=12.0 K 2024-04-14T11:17:28,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/9d999046b1304d159d6265cc513e5531 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/9d999046b1304d159d6265cc513e5531 2024-04-14T11:17:28,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/9d999046b1304d159d6265cc513e5531, entries=150, sequenceid=447, filesize=12.0 K 2024-04-14T11:17:28,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 893136536711aed76a1c4f901f8c1645 in 459ms, sequenceid=447, compaction requested=true 2024-04-14T11:17:28,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:28,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:28,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:28,197 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:28,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:28,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:28,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 893136536711aed76a1c4f901f8c1645:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:28,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:28,197 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:28,198 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39634 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:28,199 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/A is initiating minor compaction (all files) 2024-04-14T11:17:28,199 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/A in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:28,199 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37194 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:28,199 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/B is initiating minor compaction (all files) 2024-04-14T11:17:28,199 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/78102fdd10b74c519fda5efc489a55e6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/43596f20781b40f894a48d3cdc3fb898, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/28c09d6567314c07aa5af3128c77fbc8] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=38.7 K 2024-04-14T11:17:28,199 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/B in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:28,199 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/eb2f22e471b546279cd4dc6b5eb5eb24, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/623fe7b1ff1949719425cf51a3f13939, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7466a5145dee483ea7126a9d51982086] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=36.3 K 2024-04-14T11:17:28,199 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 78102fdd10b74c519fda5efc489a55e6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1713093445199 2024-04-14T11:17:28,199 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting eb2f22e471b546279cd4dc6b5eb5eb24, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1713093445199 2024-04-14T11:17:28,200 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 43596f20781b40f894a48d3cdc3fb898, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1713093446429 2024-04-14T11:17:28,200 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 623fe7b1ff1949719425cf51a3f13939, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1713093446429 2024-04-14T11:17:28,201 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 28c09d6567314c07aa5af3128c77fbc8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1713093447082 2024-04-14T11:17:28,201 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 7466a5145dee483ea7126a9d51982086, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1713093447114 2024-04-14T11:17:28,209 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#A#compaction#535 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:28,210 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/7f4f54671b1e43be9a28566840ebb184 is 50, key is test_row_0/A:col10/1713093447114/Put/seqid=0 2024-04-14T11:17:28,212 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#B#compaction#536 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:28,212 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/b3fe42f35b254cc3b3b6d1a109be020d is 50, key is test_row_0/B:col10/1713093447114/Put/seqid=0 2024-04-14T11:17:28,224 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/7f4f54671b1e43be9a28566840ebb184 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/7f4f54671b1e43be9a28566840ebb184 2024-04-14T11:17:28,226 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/b3fe42f35b254cc3b3b6d1a109be020d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b3fe42f35b254cc3b3b6d1a109be020d 2024-04-14T11:17:28,231 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/A of 893136536711aed76a1c4f901f8c1645 into 7f4f54671b1e43be9a28566840ebb184(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:28,231 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:28,231 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/A, priority=13, startTime=1713093448197; duration=0sec 2024-04-14T11:17:28,231 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:28,231 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:A 2024-04-14T11:17:28,231 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:28,233 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37194 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:28,233 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): 893136536711aed76a1c4f901f8c1645/C is initiating minor compaction (all files) 2024-04-14T11:17:28,233 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of 893136536711aed76a1c4f901f8c1645/C in TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:28,233 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/b14d157056234880a23448f6612177ec, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/2b30c86af373414db5c2ce583d5f6161, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/9d999046b1304d159d6265cc513e5531] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp, totalSize=36.3 K 2024-04-14T11:17:28,234 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting b14d157056234880a23448f6612177ec, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1713093445199 2024-04-14T11:17:28,234 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 2b30c86af373414db5c2ce583d5f6161, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1713093446429 2024-04-14T11:17:28,235 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/B of 893136536711aed76a1c4f901f8c1645 into b3fe42f35b254cc3b3b6d1a109be020d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:28,235 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:28,235 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/B, priority=13, startTime=1713093448197; duration=0sec 2024-04-14T11:17:28,235 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:28,235 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:B 2024-04-14T11:17:28,235 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 9d999046b1304d159d6265cc513e5531, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1713093447114 2024-04-14T11:17:28,241 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 893136536711aed76a1c4f901f8c1645#C#compaction#537 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:28,242 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/367e20f42fe94b41a81d1ac955ba2c6e is 50, key is test_row_0/C:col10/1713093447114/Put/seqid=0 2024-04-14T11:17:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:28,461 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:28,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-04-14T11:17:28,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/f6306416826e420e847a43eb27d64299 is 50, key is test_row_0/A:col10/1713093448460/Put/seqid=0 2024-04-14T11:17:28,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:28,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093508499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:28,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093508604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=156 2024-04-14T11:17:28,640 INFO [Thread-2284 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 156 completed 2024-04-14T11:17:28,642 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-04-14T11:17:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=158 2024-04-14T11:17:28,645 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:28,646 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:28,646 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:28,652 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/367e20f42fe94b41a81d1ac955ba2c6e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/367e20f42fe94b41a81d1ac955ba2c6e 2024-04-14T11:17:28,658 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in 893136536711aed76a1c4f901f8c1645/C of 893136536711aed76a1c4f901f8c1645 into 367e20f42fe94b41a81d1ac955ba2c6e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:28,658 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:28,658 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., storeName=893136536711aed76a1c4f901f8c1645/C, priority=13, startTime=1713093448197; duration=0sec 2024-04-14T11:17:28,659 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:28,659 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 893136536711aed76a1c4f901f8c1645:C 2024-04-14T11:17:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=158 2024-04-14T11:17:28,798 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:28,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-04-14T11:17:28,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:28,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:28,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:28,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(53): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:28,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:28,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093508808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:28,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/f6306416826e420e847a43eb27d64299 2024-04-14T11:17:28,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/051eb873c8bc45a7881c4b33b29e7a84 is 50, key is test_row_0/B:col10/1713093448460/Put/seqid=0 2024-04-14T11:17:28,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49590 deadline: 1713093508886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:28,889 DEBUG [Thread-2276 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8152 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., hostname=jenkins-hbase6.apache.org,33239,1713093321027, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2024-04-14T11:17:28,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49598 deadline: 1713093508898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:28,902 DEBUG [Thread-2282 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8159 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., hostname=jenkins-hbase6.apache.org,33239,1713093321027, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2024-04-14T11:17:28,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:28,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49600 deadline: 1713093508923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:28,927 DEBUG [Thread-2274 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8191 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., hostname=jenkins-hbase6.apache.org,33239,1713093321027, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2024-04-14T11:17:28,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49570 deadline: 1713093508930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:28,933 DEBUG [Thread-2278 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8189 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645., hostname=jenkins-hbase6.apache.org,33239,1713093321027, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2024-04-14T11:17:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=158 2024-04-14T11:17:28,951 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:28,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-04-14T11:17:28,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:28,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:28,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:28,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(53): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:28,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:29,104 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:29,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-04-14T11:17:29,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:29,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:29,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:29,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(53): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:29,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:29,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:49528 deadline: 1713093509111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=158 2024-04-14T11:17:29,257 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:29,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-04-14T11:17:29,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:29,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:29,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:29,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(53): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:29,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:29,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/051eb873c8bc45a7881c4b33b29e7a84 2024-04-14T11:17:29,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/3c47863caf334ce8aa5b6a41191c3207 is 50, key is test_row_0/C:col10/1713093448460/Put/seqid=0 2024-04-14T11:17:29,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/3c47863caf334ce8aa5b6a41191c3207 2024-04-14T11:17:29,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/f6306416826e420e847a43eb27d64299 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/f6306416826e420e847a43eb27d64299 2024-04-14T11:17:29,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/f6306416826e420e847a43eb27d64299, entries=200, sequenceid=473, filesize=14.4 K 2024-04-14T11:17:29,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/051eb873c8bc45a7881c4b33b29e7a84 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/051eb873c8bc45a7881c4b33b29e7a84 2024-04-14T11:17:29,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/051eb873c8bc45a7881c4b33b29e7a84, entries=150, sequenceid=473, filesize=12.0 K 2024-04-14T11:17:29,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/3c47863caf334ce8aa5b6a41191c3207 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/3c47863caf334ce8aa5b6a41191c3207 2024-04-14T11:17:29,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/3c47863caf334ce8aa5b6a41191c3207, entries=150, sequenceid=473, filesize=12.0 K 2024-04-14T11:17:29,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 893136536711aed76a1c4f901f8c1645 in 877ms, sequenceid=473, compaction requested=false 2024-04-14T11:17:29,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:29,408 DEBUG [Thread-2291 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7d3aa52c to 127.0.0.1:60913 2024-04-14T11:17:29,408 DEBUG [Thread-2291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:29,409 DEBUG [Thread-2289 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x47b46ade to 127.0.0.1:60913 2024-04-14T11:17:29,409 DEBUG [Thread-2289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:29,410 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:29,410 DEBUG [Thread-2287 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7106cfe7 to 127.0.0.1:60913 2024-04-14T11:17:29,410 DEBUG [Thread-2287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:29,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-04-14T11:17:29,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:29,411 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-04-14T11:17:29,411 DEBUG [Thread-2293 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x31165428 to 127.0.0.1:60913 2024-04-14T11:17:29,411 DEBUG [Thread-2293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:29,412 DEBUG [Thread-2285 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x289884d7 to 127.0.0.1:60913 2024-04-14T11:17:29,412 DEBUG [Thread-2285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:29,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/275648f240174ebfa207ae835a50e5fc is 50, key is test_row_0/A:col10/1713093448481/Put/seqid=0 2024-04-14T11:17:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:29,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. as already flushing 2024-04-14T11:17:29,621 DEBUG [Thread-2280 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2539c0e3 to 127.0.0.1:60913 2024-04-14T11:17:29,621 DEBUG [Thread-2280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=158 2024-04-14T11:17:29,820 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/275648f240174ebfa207ae835a50e5fc 2024-04-14T11:17:29,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/1e085721cc294479a7b8908d13dc5e92 is 50, key is test_row_0/B:col10/1713093448481/Put/seqid=0 2024-04-14T11:17:29,831 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/1e085721cc294479a7b8908d13dc5e92 2024-04-14T11:17:29,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/9f84e610a0394c7b9f38162e96ecc044 is 50, key is test_row_0/C:col10/1713093448481/Put/seqid=0 2024-04-14T11:17:30,242 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/9f84e610a0394c7b9f38162e96ecc044 2024-04-14T11:17:30,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/275648f240174ebfa207ae835a50e5fc as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/275648f240174ebfa207ae835a50e5fc 2024-04-14T11:17:30,252 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/275648f240174ebfa207ae835a50e5fc, entries=150, sequenceid=486, filesize=12.0 K 2024-04-14T11:17:30,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/1e085721cc294479a7b8908d13dc5e92 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/1e085721cc294479a7b8908d13dc5e92 2024-04-14T11:17:30,257 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/1e085721cc294479a7b8908d13dc5e92, entries=150, sequenceid=486, filesize=12.0 K 2024-04-14T11:17:30,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/9f84e610a0394c7b9f38162e96ecc044 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/9f84e610a0394c7b9f38162e96ecc044 2024-04-14T11:17:30,263 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/9f84e610a0394c7b9f38162e96ecc044, entries=150, sequenceid=486, filesize=12.0 K 2024-04-14T11:17:30,264 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3035): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=6.71 KB/6870 for 893136536711aed76a1c4f901f8c1645 in 852ms, sequenceid=486, compaction requested=true 2024-04-14T11:17:30,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2533): Flush status journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:30,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:30,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=159 2024-04-14T11:17:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=159 2024-04-14T11:17:30,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=159, resume processing ppid=158 2024-04-14T11:17:30,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=159, ppid=158, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6190 sec 2024-04-14T11:17:30,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=158, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.6240 sec 2024-04-14T11:17:30,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=158 2024-04-14T11:17:30,751 INFO [Thread-2284 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 158 completed 2024-04-14T11:17:38,938 DEBUG [Thread-2282 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x14e170b5 to 127.0.0.1:60913 2024-04-14T11:17:38,938 DEBUG [Thread-2282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:38,959 DEBUG [Thread-2278 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x444c7fb3 to 127.0.0.1:60913 2024-04-14T11:17:38,959 DEBUG [Thread-2278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:38,963 DEBUG [Thread-2276 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7fb749ac to 127.0.0.1:60913 2024-04-14T11:17:38,963 DEBUG [Thread-2276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:39,016 DEBUG [Thread-2274 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x523efeac to 127.0.0.1:60913 2024-04-14T11:17:39,016 DEBUG [Thread-2274 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:39,017 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-04-14T11:17:39,017 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-04-14T11:17:39,017 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-04-14T11:17:39,017 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-04-14T11:17:39,017 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 135 2024-04-14T11:17:39,017 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-04-14T11:17:39,017 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-04-14T11:17:39,017 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-04-14T11:17:39,017 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(402): scanned 2940 2024-04-14T11:17:39,017 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(403): verified 8817 rows 2024-04-14T11:17:39,017 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(402): scanned 2935 2024-04-14T11:17:39,017 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(403): verified 8805 rows 2024-04-14T11:17:39,017 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(402): scanned 2933 2024-04-14T11:17:39,018 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(403): verified 8799 rows 2024-04-14T11:17:39,018 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(402): scanned 2926 2024-04-14T11:17:39,018 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(403): verified 8776 rows 2024-04-14T11:17:39,018 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(402): scanned 2918 2024-04-14T11:17:39,018 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(403): verified 8754 rows 2024-04-14T11:17:39,018 INFO [Listener at localhost/34025 {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-04-14T11:17:39,018 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x63a6b404 to 127.0.0.1:60913 2024-04-14T11:17:39,018 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:17:39,022 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$18(968): Started disable of TestAcidGuarantees 2024-04-14T11:17:39,022 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$13(2744): Client=jenkins//172.31.2.20 disable TestAcidGuarantees 2024-04-14T11:17:39,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=160, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-04-14T11:17:39,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=160 2024-04-14T11:17:39,031 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093459031"}]},"ts":"1713093459031"} 2024-04-14T11:17:39,032 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-04-14T11:17:39,037 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set TestAcidGuarantees to state=DISABLING 2024-04-14T11:17:39,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=893136536711aed76a1c4f901f8c1645, UNASSIGN}] 2024-04-14T11:17:39,040 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(729): Took xlock for pid=161, ppid=160, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=893136536711aed76a1c4f901f8c1645, UNASSIGN 2024-04-14T11:17:39,040 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=161 updating hbase:meta row=893136536711aed76a1c4f901f8c1645, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:39,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE; CloseRegionProcedure 893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:17:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=160 2024-04-14T11:17:39,195 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:39,196 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] handler.UnassignRegionHandler(119): Close 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:39,196 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1676): Closing 893136536711aed76a1c4f901f8c1645, disabling compactions & flushes 2024-04-14T11:17:39,196 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1698): Closing region TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:39,196 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1719): Waiting without time limit for close lock on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:39,196 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1786): Acquired close lock on TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. after waiting 0 ms 2024-04-14T11:17:39,196 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1796): Updates disabled for region TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:39,196 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(2832): Flushing 893136536711aed76a1c4f901f8c1645 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-04-14T11:17:39,204 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/deead5e02d5e4348a1d2c46ce28def3f is 50, key is test_row_0/A:col10/1713093458961/Put/seqid=0 2024-04-14T11:17:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=160 2024-04-14T11:17:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=160 2024-04-14T11:17:39,637 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/deead5e02d5e4348a1d2c46ce28def3f 2024-04-14T11:17:39,647 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/f792bc2b180b4cab96d0d8838bf3407d is 50, key is test_row_0/B:col10/1713093458961/Put/seqid=0 2024-04-14T11:17:39,652 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/f792bc2b180b4cab96d0d8838bf3407d 2024-04-14T11:17:39,662 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/58f1907abb3847e39fe465afd02f63cd is 50, key is test_row_0/C:col10/1713093458961/Put/seqid=0 2024-04-14T11:17:39,668 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/58f1907abb3847e39fe465afd02f63cd 2024-04-14T11:17:39,675 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/A/deead5e02d5e4348a1d2c46ce28def3f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/deead5e02d5e4348a1d2c46ce28def3f 2024-04-14T11:17:39,681 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/deead5e02d5e4348a1d2c46ce28def3f, entries=150, sequenceid=494, filesize=12.0 K 2024-04-14T11:17:39,682 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/B/f792bc2b180b4cab96d0d8838bf3407d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/f792bc2b180b4cab96d0d8838bf3407d 2024-04-14T11:17:39,700 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/f792bc2b180b4cab96d0d8838bf3407d, entries=150, sequenceid=494, filesize=12.0 K 2024-04-14T11:17:39,701 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/.tmp/C/58f1907abb3847e39fe465afd02f63cd as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/58f1907abb3847e39fe465afd02f63cd 2024-04-14T11:17:39,708 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/58f1907abb3847e39fe465afd02f63cd, entries=150, sequenceid=494, filesize=12.0 K 2024-04-14T11:17:39,710 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(3035): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 893136536711aed76a1c4f901f8c1645 in 514ms, sequenceid=494, compaction requested=true 2024-04-14T11:17:39,710 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2d3f47b9c8bd484cb4a5c1bd79b00e45, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2e52403eed5c40ff840867d0f1e7e4fe, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/ce722c3d0f334fb1b1207d62c8c1a10f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/7a85f99463f24db5a08a58a2156fa509, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/bc9b3e67717249bc9cfa9f38e481e56d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/72a57a6f83f14b55bd271b873854df3e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/93db23979b9c48ce8b13498207e8ebba, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/32e48cb8270940018dd41f00e63e6fab, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/b997b725aa984e11a823fcc125f962f6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/78102fdd10b74c519fda5efc489a55e6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/88d630e6719b45a99bb163f969c4fe70, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/43596f20781b40f894a48d3cdc3fb898, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/28c09d6567314c07aa5af3128c77fbc8] to archive 2024-04-14T11:17:39,711 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:17:39,714 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2d3f47b9c8bd484cb4a5c1bd79b00e45 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2d3f47b9c8bd484cb4a5c1bd79b00e45 2024-04-14T11:17:39,716 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2e52403eed5c40ff840867d0f1e7e4fe to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/2e52403eed5c40ff840867d0f1e7e4fe 2024-04-14T11:17:39,718 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/ce722c3d0f334fb1b1207d62c8c1a10f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/ce722c3d0f334fb1b1207d62c8c1a10f 2024-04-14T11:17:39,720 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/7a85f99463f24db5a08a58a2156fa509 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/7a85f99463f24db5a08a58a2156fa509 2024-04-14T11:17:39,722 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/bc9b3e67717249bc9cfa9f38e481e56d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/bc9b3e67717249bc9cfa9f38e481e56d 2024-04-14T11:17:39,724 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/72a57a6f83f14b55bd271b873854df3e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/72a57a6f83f14b55bd271b873854df3e 2024-04-14T11:17:39,726 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/93db23979b9c48ce8b13498207e8ebba to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/93db23979b9c48ce8b13498207e8ebba 2024-04-14T11:17:39,727 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/32e48cb8270940018dd41f00e63e6fab to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/32e48cb8270940018dd41f00e63e6fab 2024-04-14T11:17:39,729 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/b997b725aa984e11a823fcc125f962f6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/b997b725aa984e11a823fcc125f962f6 2024-04-14T11:17:39,730 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/78102fdd10b74c519fda5efc489a55e6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/78102fdd10b74c519fda5efc489a55e6 2024-04-14T11:17:39,732 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/88d630e6719b45a99bb163f969c4fe70 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/88d630e6719b45a99bb163f969c4fe70 2024-04-14T11:17:39,734 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/43596f20781b40f894a48d3cdc3fb898 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/43596f20781b40f894a48d3cdc3fb898 2024-04-14T11:17:39,736 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/28c09d6567314c07aa5af3128c77fbc8 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/28c09d6567314c07aa5af3128c77fbc8 2024-04-14T11:17:39,737 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/bd95ff9ceb6148c5a93338dff5c2a3ec, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/db81a169aa1e4e1383103d56042ff2bd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/460e7de1764c477f80795589a518f3ce, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/1f94b361502f4d5fb6d58f55932102cf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b0268e2915574bf981abfddd322f0005, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/ebf5050c956641529112cc35819a7908, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/141acf68b9fc4f7481678ea54aeab117, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/85efeb386e77495c9957f164ec03320d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b5c7a5dc0cc54962b06df0dbab9230c7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/eb2f22e471b546279cd4dc6b5eb5eb24, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/91ad8b999cc34ea29908addd2ee85979, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/623fe7b1ff1949719425cf51a3f13939, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7466a5145dee483ea7126a9d51982086] to archive 2024-04-14T11:17:39,738 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:17:39,741 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/bd95ff9ceb6148c5a93338dff5c2a3ec to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/bd95ff9ceb6148c5a93338dff5c2a3ec 2024-04-14T11:17:39,743 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/db81a169aa1e4e1383103d56042ff2bd to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/db81a169aa1e4e1383103d56042ff2bd 2024-04-14T11:17:39,745 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/460e7de1764c477f80795589a518f3ce to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/460e7de1764c477f80795589a518f3ce 2024-04-14T11:17:39,746 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/1f94b361502f4d5fb6d58f55932102cf to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/1f94b361502f4d5fb6d58f55932102cf 2024-04-14T11:17:39,748 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b0268e2915574bf981abfddd322f0005 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b0268e2915574bf981abfddd322f0005 2024-04-14T11:17:39,750 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/ebf5050c956641529112cc35819a7908 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/ebf5050c956641529112cc35819a7908 2024-04-14T11:17:39,752 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/141acf68b9fc4f7481678ea54aeab117 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/141acf68b9fc4f7481678ea54aeab117 2024-04-14T11:17:39,754 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/85efeb386e77495c9957f164ec03320d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/85efeb386e77495c9957f164ec03320d 2024-04-14T11:17:39,756 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b5c7a5dc0cc54962b06df0dbab9230c7 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b5c7a5dc0cc54962b06df0dbab9230c7 2024-04-14T11:17:39,759 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/eb2f22e471b546279cd4dc6b5eb5eb24 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/eb2f22e471b546279cd4dc6b5eb5eb24 2024-04-14T11:17:39,761 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/91ad8b999cc34ea29908addd2ee85979 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/91ad8b999cc34ea29908addd2ee85979 2024-04-14T11:17:39,763 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/623fe7b1ff1949719425cf51a3f13939 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/623fe7b1ff1949719425cf51a3f13939 2024-04-14T11:17:39,765 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7466a5145dee483ea7126a9d51982086 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/7466a5145dee483ea7126a9d51982086 2024-04-14T11:17:39,767 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/8d29d736fbe24798bd6f45882f203370, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/14380760189c4ff29c33dac91f3bc2e1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/c8c8bc2fee7e4b529cbbb12237aac1e1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/8112e1f8abb449c5afa67564607d8c39, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/8c8f9305f7894c8a9ad934ee85a45565, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/fbd30c21b91947d790e41a78a072eead, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/1f32bcb4eafb4e69853bff88638eea3d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a7d7a55b080b4a2f8e28ebafadcb0ada, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a098e9ba685c4cdeb36cf242e79f7592, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/b14d157056234880a23448f6612177ec, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/75d9b5c4b86b4deeb407c9d07c4cdc37, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/2b30c86af373414db5c2ce583d5f6161, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/9d999046b1304d159d6265cc513e5531] to archive 2024-04-14T11:17:39,767 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:17:39,770 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/8d29d736fbe24798bd6f45882f203370 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/8d29d736fbe24798bd6f45882f203370 2024-04-14T11:17:39,772 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/14380760189c4ff29c33dac91f3bc2e1 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/14380760189c4ff29c33dac91f3bc2e1 2024-04-14T11:17:39,774 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/c8c8bc2fee7e4b529cbbb12237aac1e1 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/c8c8bc2fee7e4b529cbbb12237aac1e1 2024-04-14T11:17:39,776 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/8112e1f8abb449c5afa67564607d8c39 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/8112e1f8abb449c5afa67564607d8c39 2024-04-14T11:17:39,778 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/8c8f9305f7894c8a9ad934ee85a45565 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/8c8f9305f7894c8a9ad934ee85a45565 2024-04-14T11:17:39,785 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/fbd30c21b91947d790e41a78a072eead to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/fbd30c21b91947d790e41a78a072eead 2024-04-14T11:17:39,787 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/1f32bcb4eafb4e69853bff88638eea3d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/1f32bcb4eafb4e69853bff88638eea3d 2024-04-14T11:17:39,788 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a7d7a55b080b4a2f8e28ebafadcb0ada to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a7d7a55b080b4a2f8e28ebafadcb0ada 2024-04-14T11:17:39,790 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a098e9ba685c4cdeb36cf242e79f7592 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/a098e9ba685c4cdeb36cf242e79f7592 2024-04-14T11:17:39,792 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/b14d157056234880a23448f6612177ec to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/b14d157056234880a23448f6612177ec 2024-04-14T11:17:39,794 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/75d9b5c4b86b4deeb407c9d07c4cdc37 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/75d9b5c4b86b4deeb407c9d07c4cdc37 2024-04-14T11:17:39,796 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/2b30c86af373414db5c2ce583d5f6161 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/2b30c86af373414db5c2ce583d5f6161 2024-04-14T11:17:39,798 DEBUG [StoreCloser-TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/9d999046b1304d159d6265cc513e5531 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/9d999046b1304d159d6265cc513e5531 2024-04-14T11:17:39,804 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/recovered.edits/497.seqid, newMaxSeqId=497, maxSeqId=1 2024-04-14T11:17:39,806 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1917): Closed TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645. 2024-04-14T11:17:39,806 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1630): Region close journal for 893136536711aed76a1c4f901f8c1645: 2024-04-14T11:17:39,808 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] handler.UnassignRegionHandler(163): Closed 893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:39,809 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=161 updating hbase:meta row=893136536711aed76a1c4f901f8c1645, regionState=CLOSED 2024-04-14T11:17:39,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=162, resume processing ppid=161 2024-04-14T11:17:39,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=162, ppid=161, state=SUCCESS; CloseRegionProcedure 893136536711aed76a1c4f901f8c1645, server=jenkins-hbase6.apache.org,33239,1713093321027 in 771 msec 2024-04-14T11:17:39,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=161, resume processing ppid=160 2024-04-14T11:17:39,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=161, ppid=160, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=893136536711aed76a1c4f901f8c1645, UNASSIGN in 779 msec 2024-04-14T11:17:39,819 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093459819"}]},"ts":"1713093459819"} 2024-04-14T11:17:39,821 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-04-14T11:17:39,824 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set TestAcidGuarantees to state=DISABLED 2024-04-14T11:17:39,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=160, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 803 msec 2024-04-14T11:17:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=160 2024-04-14T11:17:40,136 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$TableFuture(3746): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 160 completed 2024-04-14T11:17:40,137 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$5(2494): Client=jenkins//172.31.2.20 delete TestAcidGuarantees 2024-04-14T11:17:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=163, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:17:40,141 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=163, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:17:40,142 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=163, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:17:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=163 2024-04-14T11:17:40,153 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:40,159 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A, FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B, FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C, FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/recovered.edits] 2024-04-14T11:17:40,165 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/275648f240174ebfa207ae835a50e5fc to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/275648f240174ebfa207ae835a50e5fc 2024-04-14T11:17:40,167 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/7f4f54671b1e43be9a28566840ebb184 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/7f4f54671b1e43be9a28566840ebb184 2024-04-14T11:17:40,169 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/deead5e02d5e4348a1d2c46ce28def3f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/deead5e02d5e4348a1d2c46ce28def3f 2024-04-14T11:17:40,171 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/f6306416826e420e847a43eb27d64299 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/A/f6306416826e420e847a43eb27d64299 2024-04-14T11:17:40,176 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/051eb873c8bc45a7881c4b33b29e7a84 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/051eb873c8bc45a7881c4b33b29e7a84 2024-04-14T11:17:40,177 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/1e085721cc294479a7b8908d13dc5e92 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/1e085721cc294479a7b8908d13dc5e92 2024-04-14T11:17:40,179 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b3fe42f35b254cc3b3b6d1a109be020d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/b3fe42f35b254cc3b3b6d1a109be020d 2024-04-14T11:17:40,182 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/f792bc2b180b4cab96d0d8838bf3407d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/B/f792bc2b180b4cab96d0d8838bf3407d 2024-04-14T11:17:40,185 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/367e20f42fe94b41a81d1ac955ba2c6e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/367e20f42fe94b41a81d1ac955ba2c6e 2024-04-14T11:17:40,186 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/3c47863caf334ce8aa5b6a41191c3207 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/3c47863caf334ce8aa5b6a41191c3207 2024-04-14T11:17:40,187 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/58f1907abb3847e39fe465afd02f63cd to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/58f1907abb3847e39fe465afd02f63cd 2024-04-14T11:17:40,188 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/9f84e610a0394c7b9f38162e96ecc044 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/C/9f84e610a0394c7b9f38162e96ecc044 2024-04-14T11:17:40,196 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/recovered.edits/497.seqid to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645/recovered.edits/497.seqid 2024-04-14T11:17:40,200 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/893136536711aed76a1c4f901f8c1645 2024-04-14T11:17:40,200 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived TestAcidGuarantees regions 2024-04-14T11:17:40,203 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=163, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:17:40,219 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-04-14T11:17:40,221 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'TestAcidGuarantees' descriptor. 2024-04-14T11:17:40,223 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=163, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:17:40,223 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'TestAcidGuarantees' from region states. 2024-04-14T11:17:40,223 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1713093460223"}]},"ts":"9223372036854775807"} 2024-04-14T11:17:40,225 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-04-14T11:17:40,225 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 893136536711aed76a1c4f901f8c1645, NAME => 'TestAcidGuarantees,,1713093428730.893136536711aed76a1c4f901f8c1645.', STARTKEY => '', ENDKEY => ''}] 2024-04-14T11:17:40,225 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'TestAcidGuarantees' as deleted. 2024-04-14T11:17:40,225 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1713093460225"}]},"ts":"9223372036854775807"} 2024-04-14T11:17:40,227 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-04-14T11:17:40,230 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(127): Finished pid=163, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:17:40,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=163, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 94 msec 2024-04-14T11:17:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=163 2024-04-14T11:17:40,243 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$TableFuture(3746): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 163 completed 2024-04-14T11:17:40,263 INFO [Listener at localhost/34025 {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=220 (was 223), OpenFileDescriptor=510 (was 530), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=486 (was 588), ProcessCount=168 (was 168), AvailableMemoryMB=1474 (was 1561) 2024-04-14T11:17:40,280 INFO [Listener at localhost/34025 {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=220, OpenFileDescriptor=510, MaxFileDescriptor=60000, SystemLoadAverage=486, ProcessCount=168, AvailableMemoryMB=1473 2024-04-14T11:17:40,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-04-14T11:17:40,284 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$4(2378): Client=jenkins//172.31.2.20 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-04-14T11:17:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=164, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-04-14T11:17:40,287 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(84): pid=164, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-04-14T11:17:40,287 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:17:40,288 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 164 2024-04-14T11:17:40,288 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(84): pid=164, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-04-14T11:17:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=164 2024-04-14T11:17:40,300 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7088): creating {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a 2024-04-14T11:17:40,309 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(891): Instantiated TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:17:40,309 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Closing d2b4a72b9f8ab01eabcbb9f313e8e868, disabling compactions & flushes 2024-04-14T11:17:40,310 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1698): Closing region TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:40,310 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1719): Waiting without time limit for close lock on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:40,310 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1786): Acquired close lock on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. after waiting 0 ms 2024-04-14T11:17:40,310 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1796): Updates disabled for region TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:40,310 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1917): Closed TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:40,310 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1630): Region close journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:40,311 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(84): pid=164, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-04-14T11:17:40,311 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1713093460311"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1713093460311"}]},"ts":"1713093460311"} 2024-04-14T11:17:40,313 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-04-14T11:17:40,314 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(84): pid=164, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-04-14T11:17:40,314 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093460314"}]},"ts":"1713093460314"} 2024-04-14T11:17:40,315 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-04-14T11:17:40,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2b4a72b9f8ab01eabcbb9f313e8e868, ASSIGN}] 2024-04-14T11:17:40,322 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(729): Took xlock for pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2b4a72b9f8ab01eabcbb9f313e8e868, ASSIGN 2024-04-14T11:17:40,323 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2b4a72b9f8ab01eabcbb9f313e8e868, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,33239,1713093321027; forceNewPlan=false, retain=false 2024-04-14T11:17:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=164 2024-04-14T11:17:40,474 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=165 updating hbase:meta row=d2b4a72b9f8ab01eabcbb9f313e8e868, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:40,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE; OpenRegionProcedure d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:17:40,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=164 2024-04-14T11:17:40,628 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:40,631 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=166}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:40,632 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=166}] regionserver.HRegion(7267): Opening region: {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} 2024-04-14T11:17:40,632 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=166}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:40,632 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=166}] regionserver.HRegion(891): Instantiated TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:17:40,632 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=166}] regionserver.HRegion(7309): checking encryption for d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:40,632 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=166}] regionserver.HRegion(7312): checking classloading for d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:40,633 INFO [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:40,635 INFO [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2b4a72b9f8ab01eabcbb9f313e8e868 columnFamilyName A 2024-04-14T11:17:40,635 DEBUG [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:17:40,635 INFO [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] regionserver.HStore(324): Store=d2b4a72b9f8ab01eabcbb9f313e8e868/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:17:40,636 INFO [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:40,637 INFO [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2b4a72b9f8ab01eabcbb9f313e8e868 columnFamilyName B 2024-04-14T11:17:40,637 DEBUG [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:17:40,637 INFO [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] regionserver.HStore(324): Store=d2b4a72b9f8ab01eabcbb9f313e8e868/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:17:40,638 INFO [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:40,639 INFO [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2b4a72b9f8ab01eabcbb9f313e8e868 columnFamilyName C 2024-04-14T11:17:40,639 DEBUG [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:17:40,639 INFO [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] regionserver.HStore(324): Store=d2b4a72b9f8ab01eabcbb9f313e8e868/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:17:40,640 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=166}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:40,640 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=166}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:40,642 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=166}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-04-14T11:17:40,644 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=166}] regionserver.HRegion(1082): writing seq id for d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:40,646 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=166}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-04-14T11:17:40,646 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=166}] regionserver.HRegion(1099): Opened d2b4a72b9f8ab01eabcbb9f313e8e868; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61879169, jitterRate=-0.07792852818965912}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-04-14T11:17:40,647 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=166}] regionserver.HRegion(998): Region open journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:40,647 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=166}] regionserver.HRegionServer(2593): Post open deploy tasks for TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., pid=166, masterSystemTime=1713093460628 2024-04-14T11:17:40,649 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=166}] regionserver.HRegionServer(2620): Finished post open deploy task for TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:40,649 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=166}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:40,649 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=165 updating hbase:meta row=d2b4a72b9f8ab01eabcbb9f313e8e868, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:40,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=166, resume processing ppid=165 2024-04-14T11:17:40,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=166, ppid=165, state=SUCCESS; OpenRegionProcedure d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 in 174 msec 2024-04-14T11:17:40,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=165, resume processing ppid=164 2024-04-14T11:17:40,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=165, ppid=164, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2b4a72b9f8ab01eabcbb9f313e8e868, ASSIGN in 331 msec 2024-04-14T11:17:40,654 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(84): pid=164, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-04-14T11:17:40,654 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093460654"}]},"ts":"1713093460654"} 2024-04-14T11:17:40,655 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-04-14T11:17:40,664 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(84): pid=164, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-04-14T11:17:40,665 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=164, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 380 msec 2024-04-14T11:17:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=164 2024-04-14T11:17:40,893 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$TableFuture(3746): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 164 completed 2024-04-14T11:17:40,895 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5d97b1f8 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:40,901 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1765af04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:40,903 DEBUG [Listener at localhost/34025 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:40,906 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:53632, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:40,907 DEBUG [Listener at localhost/34025 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-04-14T11:17:40,908 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:52174, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-04-14T11:17:40,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-04-14T11:17:40,911 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$14(2787): Client=jenkins//172.31.2.20 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-04-14T11:17:40,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=167, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=TestAcidGuarantees 2024-04-14T11:17:40,927 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-04-14T11:17:40,927 INFO [PEWorker-1 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-04-14T11:17:40,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-04-14T11:17:40,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2b4a72b9f8ab01eabcbb9f313e8e868, REOPEN/MOVE}] 2024-04-14T11:17:40,933 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(729): Took xlock for pid=169, ppid=168, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2b4a72b9f8ab01eabcbb9f313e8e868, REOPEN/MOVE 2024-04-14T11:17:40,934 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=169 updating hbase:meta row=d2b4a72b9f8ab01eabcbb9f313e8e868, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:40,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE; CloseRegionProcedure d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:17:41,088 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:41,088 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] handler.UnassignRegionHandler(119): Close d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:41,089 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1676): Closing d2b4a72b9f8ab01eabcbb9f313e8e868, disabling compactions & flushes 2024-04-14T11:17:41,089 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1698): Closing region TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:41,089 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1719): Waiting without time limit for close lock on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:41,089 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1786): Acquired close lock on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. after waiting 0 ms 2024-04-14T11:17:41,089 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1796): Updates disabled for region TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:41,094 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-04-14T11:17:41,095 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1917): Closed TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:41,095 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1630): Region close journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:41,095 WARN [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegionServer(3777): Not adding moved region record: d2b4a72b9f8ab01eabcbb9f313e8e868 to self. 2024-04-14T11:17:41,097 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] handler.UnassignRegionHandler(163): Closed d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:41,098 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=169 updating hbase:meta row=d2b4a72b9f8ab01eabcbb9f313e8e868, regionState=CLOSED 2024-04-14T11:17:41,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=170, resume processing ppid=169 2024-04-14T11:17:41,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=170, ppid=169, state=SUCCESS; CloseRegionProcedure d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 in 163 msec 2024-04-14T11:17:41,102 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=169, ppid=168, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2b4a72b9f8ab01eabcbb9f313e8e868, REOPEN/MOVE; state=CLOSED, location=jenkins-hbase6.apache.org,33239,1713093321027; forceNewPlan=false, retain=true 2024-04-14T11:17:41,253 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=169 updating hbase:meta row=d2b4a72b9f8ab01eabcbb9f313e8e868, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:41,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=171, ppid=169, state=RUNNABLE; OpenRegionProcedure d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:17:41,406 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:41,410 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=171}] handler.AssignRegionHandler(130): Open TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:41,410 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=171}] regionserver.HRegion(7267): Opening region: {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} 2024-04-14T11:17:41,411 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=171}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:41,411 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=171}] regionserver.HRegion(891): Instantiated TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-04-14T11:17:41,411 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=171}] regionserver.HRegion(7309): checking encryption for d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:41,411 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=171}] regionserver.HRegion(7312): checking classloading for d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:41,414 INFO [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:41,415 INFO [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2b4a72b9f8ab01eabcbb9f313e8e868 columnFamilyName A 2024-04-14T11:17:41,417 DEBUG [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:17:41,420 INFO [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] regionserver.HStore(324): Store=d2b4a72b9f8ab01eabcbb9f313e8e868/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:17:41,421 INFO [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:41,422 INFO [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2b4a72b9f8ab01eabcbb9f313e8e868 columnFamilyName B 2024-04-14T11:17:41,422 DEBUG [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:17:41,422 INFO [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] regionserver.HStore(324): Store=d2b4a72b9f8ab01eabcbb9f313e8e868/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:17:41,423 INFO [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:41,423 INFO [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2b4a72b9f8ab01eabcbb9f313e8e868 columnFamilyName C 2024-04-14T11:17:41,423 DEBUG [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-04-14T11:17:41,424 INFO [StoreOpener-d2b4a72b9f8ab01eabcbb9f313e8e868-1 {}] regionserver.HStore(324): Store=d2b4a72b9f8ab01eabcbb9f313e8e868/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-04-14T11:17:41,425 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=171}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:41,426 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=171}] regionserver.HRegion(5283): Found 0 recovered edits file(s) under hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:41,428 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=171}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-04-14T11:17:41,431 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=171}] regionserver.HRegion(1082): writing seq id for d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:41,432 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=171}] regionserver.HRegion(1099): Opened d2b4a72b9f8ab01eabcbb9f313e8e868; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63031828, jitterRate=-0.06075257062911987}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-04-14T11:17:41,432 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=171}] regionserver.HRegion(998): Region open journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:41,433 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=171}] regionserver.HRegionServer(2593): Post open deploy tasks for TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., pid=171, masterSystemTime=1713093461406 2024-04-14T11:17:41,435 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=171}] regionserver.HRegionServer(2620): Finished post open deploy task for TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:41,435 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=171}] handler.AssignRegionHandler(158): Opened TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:41,436 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=169 updating hbase:meta row=d2b4a72b9f8ab01eabcbb9f313e8e868, regionState=OPEN, openSeqNum=5, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:41,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=171, resume processing ppid=169 2024-04-14T11:17:41,440 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=171, ppid=169, state=SUCCESS; OpenRegionProcedure d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 in 184 msec 2024-04-14T11:17:41,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=169, resume processing ppid=168 2024-04-14T11:17:41,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=169, ppid=168, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2b4a72b9f8ab01eabcbb9f313e8e868, REOPEN/MOVE in 508 msec 2024-04-14T11:17:41,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=168, resume processing ppid=167 2024-04-14T11:17:41,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=168, ppid=167, state=SUCCESS; ReopenTableRegionsProcedure table=TestAcidGuarantees in 512 msec 2024-04-14T11:17:41,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=167, state=SUCCESS; ModifyTableProcedure table=TestAcidGuarantees in 534 msec 2024-04-14T11:17:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=167 2024-04-14T11:17:41,450 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x170f16e4 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:41,476 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f77d8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:41,477 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x53f6ff98 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:41,487 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c216b34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:41,488 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3a6302ef to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:41,497 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e0d2004, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:41,498 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2a265051 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:41,508 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f05b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:41,510 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x42b49b11 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:41,524 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a08efce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:41,525 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5bbf00f8 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:41,539 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d4f2de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:41,540 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x74251eef to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:41,560 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57be6853, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:41,561 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x77737ebd to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:41,583 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bfdd4da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:41,584 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x50f4a068 to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:41,598 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c1eaa5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:41,599 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x58eccddb to 127.0.0.1:60913 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2024-04-14T11:17:41,617 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d6ec14e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-04-14T11:17:41,642 DEBUG [hconnection-0x71ba988c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:41,642 DEBUG [hconnection-0x23c9e44f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:41,644 DEBUG [hconnection-0x4d618748-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:41,644 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:46152, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:41,645 DEBUG [hconnection-0x230e9eaa-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:41,646 DEBUG [hconnection-0xc9f41bf-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:41,650 DEBUG [hconnection-0x519dfd14-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:41,650 DEBUG [hconnection-0x5bc958fb-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:41,651 DEBUG [hconnection-0x5f0848e8-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:41,652 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:46172, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:41,652 DEBUG [hconnection-0x69d6d2d0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:41,652 DEBUG [hconnection-0x6f144d8e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-04-14T11:17:41,656 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:46162, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:41,656 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:46186, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:41,656 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:46174, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:41,662 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:46200, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:41,668 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:46146, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:41,669 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:46170, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:41,669 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:46180, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:41,669 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.31.2.20:46214, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-04-14T11:17:41,670 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:41,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-04-14T11:17:41,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=172 2024-04-14T11:17:41,681 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:41,682 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:41,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:41,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:41,687 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:41,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-04-14T11:17:41,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:41,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093521711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:41,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093521713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:41,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093521716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:41,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093521717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:41,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093521717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:41,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404143c95ec4683a14d03a6e37a2bcb68374a_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093461687/Put/seqid=0 2024-04-14T11:17:41,772 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404143c95ec4683a14d03a6e37a2bcb68374a_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404143c95ec4683a14d03a6e37a2bcb68374a_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:41,774 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/ffb0ad2f549c4c75824d0573df1681dd, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:41,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/ffb0ad2f549c4c75824d0573df1681dd is 175, key is test_row_0/A:col10/1713093461687/Put/seqid=0 2024-04-14T11:17:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=172 2024-04-14T11:17:41,797 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/ffb0ad2f549c4c75824d0573df1681dd 2024-04-14T11:17:41,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093521819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:41,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093521821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:41,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093521821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:41,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093521821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:41,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093521821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:41,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/3ab387b3d4f7457489ed3a2412bd8788 is 50, key is test_row_0/B:col10/1713093461687/Put/seqid=0 2024-04-14T11:17:41,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/3ab387b3d4f7457489ed3a2412bd8788 2024-04-14T11:17:41,836 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:41,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-04-14T11:17:41,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:41,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:41,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:41,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(53): pid=173 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:41,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:41,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/7cf5af8c3dfd455fb98f7f6254953398 is 50, key is test_row_0/C:col10/1713093461687/Put/seqid=0 2024-04-14T11:17:41,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/7cf5af8c3dfd455fb98f7f6254953398 2024-04-14T11:17:41,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/ffb0ad2f549c4c75824d0573df1681dd as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ffb0ad2f549c4c75824d0573df1681dd 2024-04-14T11:17:41,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ffb0ad2f549c4c75824d0573df1681dd, entries=150, sequenceid=17, filesize=30.2 K 2024-04-14T11:17:41,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/3ab387b3d4f7457489ed3a2412bd8788 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/3ab387b3d4f7457489ed3a2412bd8788 2024-04-14T11:17:41,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/3ab387b3d4f7457489ed3a2412bd8788, entries=150, sequenceid=17, filesize=11.7 K 2024-04-14T11:17:41,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/7cf5af8c3dfd455fb98f7f6254953398 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/7cf5af8c3dfd455fb98f7f6254953398 2024-04-14T11:17:41,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/7cf5af8c3dfd455fb98f7f6254953398, entries=150, sequenceid=17, filesize=11.7 K 2024-04-14T11:17:41,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 246ms, sequenceid=17, compaction requested=false 2024-04-14T11:17:41,933 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-04-14T11:17:41,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=172 2024-04-14T11:17:41,990 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:41,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-04-14T11:17:41,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:41,991 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-04-14T11:17:42,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414499c2d1065e948e183cd5ea6555e2943_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093461707/Put/seqid=0 2024-04-14T11:17:42,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:42,039 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414499c2d1065e948e183cd5ea6555e2943_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414499c2d1065e948e183cd5ea6555e2943_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:42,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/be64bfb05df04348881d6e9019b5827e, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:42,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093522035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/be64bfb05df04348881d6e9019b5827e is 175, key is test_row_0/A:col10/1713093461707/Put/seqid=0 2024-04-14T11:17:42,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093522036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093522038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093522039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093522039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,054 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/be64bfb05df04348881d6e9019b5827e 2024-04-14T11:17:42,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/1fbafa4a7b504a9796c9550214b95713 is 50, key is test_row_0/B:col10/1713093461707/Put/seqid=0 2024-04-14T11:17:42,107 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/1fbafa4a7b504a9796c9550214b95713 2024-04-14T11:17:42,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/d0ffcaa00bf04b03ad76de7fcc02197f is 50, key is test_row_0/C:col10/1713093461707/Put/seqid=0 2024-04-14T11:17:42,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093522142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093522144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093522144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093522144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093522144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,157 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/d0ffcaa00bf04b03ad76de7fcc02197f 2024-04-14T11:17:42,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/be64bfb05df04348881d6e9019b5827e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/be64bfb05df04348881d6e9019b5827e 2024-04-14T11:17:42,169 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/be64bfb05df04348881d6e9019b5827e, entries=150, sequenceid=42, filesize=30.2 K 2024-04-14T11:17:42,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/1fbafa4a7b504a9796c9550214b95713 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/1fbafa4a7b504a9796c9550214b95713 2024-04-14T11:17:42,180 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/1fbafa4a7b504a9796c9550214b95713, entries=150, sequenceid=42, filesize=11.7 K 2024-04-14T11:17:42,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/d0ffcaa00bf04b03ad76de7fcc02197f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/d0ffcaa00bf04b03ad76de7fcc02197f 2024-04-14T11:17:42,191 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/d0ffcaa00bf04b03ad76de7fcc02197f, entries=150, sequenceid=42, filesize=11.7 K 2024-04-14T11:17:42,192 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3035): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 201ms, sequenceid=42, compaction requested=false 2024-04-14T11:17:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:42,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=173 2024-04-14T11:17:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=173 2024-04-14T11:17:42,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=173, resume processing ppid=172 2024-04-14T11:17:42,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=173, ppid=172, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 512 msec 2024-04-14T11:17:42,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=172, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 528 msec 2024-04-14T11:17:42,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=172 2024-04-14T11:17:42,281 INFO [Thread-2769 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 172 completed 2024-04-14T11:17:42,283 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:42,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-04-14T11:17:42,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=174 2024-04-14T11:17:42,289 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:42,291 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:42,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:42,347 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:42,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-04-14T11:17:42,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414e3f2efc7b076497cb15539ccee06984b_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093462346/Put/seqid=0 2024-04-14T11:17:42,376 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414e3f2efc7b076497cb15539ccee06984b_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414e3f2efc7b076497cb15539ccee06984b_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:42,378 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/53ddedb94ccd4b228cb44580c0f320f5, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:42,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/53ddedb94ccd4b228cb44580c0f320f5 is 175, key is test_row_0/A:col10/1713093462346/Put/seqid=0 2024-04-14T11:17:42,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093522379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093522380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093522384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093522386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093522387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=174 2024-04-14T11:17:42,403 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/53ddedb94ccd4b228cb44580c0f320f5 2024-04-14T11:17:42,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/94b10956d92642cea097f113036c6374 is 50, key is test_row_0/B:col10/1713093462346/Put/seqid=0 2024-04-14T11:17:42,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/94b10956d92642cea097f113036c6374 2024-04-14T11:17:42,446 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-04-14T11:17:42,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:42,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:42,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:42,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(53): pid=175 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:42,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:42,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/3744c3bede2f4d16a3624a2c0de97469 is 50, key is test_row_0/C:col10/1713093462346/Put/seqid=0 2024-04-14T11:17:42,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093522488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093522488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093522488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093522491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093522492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=174 2024-04-14T11:17:42,600 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-04-14T11:17:42,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:42,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:42,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:42,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(53): pid=175 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:42,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:42,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093522690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093522696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093522698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093522698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093522699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,753 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-04-14T11:17:42,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:42,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:42,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:42,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(53): pid=175 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:42,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:42,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/3744c3bede2f4d16a3624a2c0de97469 2024-04-14T11:17:42,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/53ddedb94ccd4b228cb44580c0f320f5 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/53ddedb94ccd4b228cb44580c0f320f5 2024-04-14T11:17:42,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/53ddedb94ccd4b228cb44580c0f320f5, entries=200, sequenceid=56, filesize=38.6 K 2024-04-14T11:17:42,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/94b10956d92642cea097f113036c6374 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/94b10956d92642cea097f113036c6374 2024-04-14T11:17:42,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/94b10956d92642cea097f113036c6374, entries=150, sequenceid=56, filesize=11.7 K 2024-04-14T11:17:42,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/3744c3bede2f4d16a3624a2c0de97469 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/3744c3bede2f4d16a3624a2c0de97469 2024-04-14T11:17:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=174 2024-04-14T11:17:42,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/3744c3bede2f4d16a3624a2c0de97469, entries=150, sequenceid=56, filesize=11.7 K 2024-04-14T11:17:42,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 559ms, sequenceid=56, compaction requested=true 2024-04-14T11:17:42,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:42,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:42,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:42,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:42,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:42,906 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:42,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:42,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-04-14T11:17:42,906 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:42,907 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:42,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-04-14T11:17:42,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:42,908 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-04-14T11:17:42,909 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35952 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:42,909 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/B is initiating minor compaction (all files) 2024-04-14T11:17:42,909 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/B in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:42,909 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/3ab387b3d4f7457489ed3a2412bd8788, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/1fbafa4a7b504a9796c9550214b95713, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/94b10956d92642cea097f113036c6374] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=35.1 K 2024-04-14T11:17:42,909 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101408 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:42,910 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/A is initiating minor compaction (all files) 2024-04-14T11:17:42,910 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/A in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:42,910 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ffb0ad2f549c4c75824d0573df1681dd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/be64bfb05df04348881d6e9019b5827e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/53ddedb94ccd4b228cb44580c0f320f5] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=99.0 K 2024-04-14T11:17:42,910 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:42,910 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ffb0ad2f549c4c75824d0573df1681dd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/be64bfb05df04348881d6e9019b5827e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/53ddedb94ccd4b228cb44580c0f320f5] 2024-04-14T11:17:42,910 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 3ab387b3d4f7457489ed3a2412bd8788, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1713093461683 2024-04-14T11:17:42,911 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting ffb0ad2f549c4c75824d0573df1681dd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1713093461683 2024-04-14T11:17:42,912 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 1fbafa4a7b504a9796c9550214b95713, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1713093461707 2024-04-14T11:17:42,912 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting be64bfb05df04348881d6e9019b5827e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1713093461707 2024-04-14T11:17:42,913 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 94b10956d92642cea097f113036c6374, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1713093462026 2024-04-14T11:17:42,913 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 53ddedb94ccd4b228cb44580c0f320f5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1713093462026 2024-04-14T11:17:42,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404149468e16c69704b23874b9298c3be20eb_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093462376/Put/seqid=0 2024-04-14T11:17:42,942 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:42,945 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#B#compaction#558 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:42,945 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/8a22e2b9515b42d1b89fbdeeae1ae3bb is 50, key is test_row_0/B:col10/1713093462346/Put/seqid=0 2024-04-14T11:17:42,945 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20240414e10825078c2546e2826170aa4cb86a7b_d2b4a72b9f8ab01eabcbb9f313e8e868 store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:42,947 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20240414e10825078c2546e2826170aa4cb86a7b_d2b4a72b9f8ab01eabcbb9f313e8e868, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:42,947 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414e10825078c2546e2826170aa4cb86a7b_d2b4a72b9f8ab01eabcbb9f313e8e868 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:42,970 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404149468e16c69704b23874b9298c3be20eb_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404149468e16c69704b23874b9298c3be20eb_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:42,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/14a1ab5a9a4e436b9e8f9921f19bd68e, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:42,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/14a1ab5a9a4e436b9e8f9921f19bd68e is 175, key is test_row_0/A:col10/1713093462376/Put/seqid=0 2024-04-14T11:17:42,987 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#A#compaction#557 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:42,988 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/1b0698db581140c185db13119148363a is 175, key is test_row_0/A:col10/1713093462346/Put/seqid=0 2024-04-14T11:17:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:42,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:43,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093523011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093523011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093523011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093523012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093523012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,036 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/14a1ab5a9a4e436b9e8f9921f19bd68e 2024-04-14T11:17:43,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/68b291210f4649bf84fc841400f54765 is 50, key is test_row_0/B:col10/1713093462376/Put/seqid=0 2024-04-14T11:17:43,063 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/68b291210f4649bf84fc841400f54765 2024-04-14T11:17:43,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/bb113cde374c4e4bac4d8aee8895fa62 is 50, key is test_row_0/C:col10/1713093462376/Put/seqid=0 2024-04-14T11:17:43,087 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/bb113cde374c4e4bac4d8aee8895fa62 2024-04-14T11:17:43,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/14a1ab5a9a4e436b9e8f9921f19bd68e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/14a1ab5a9a4e436b9e8f9921f19bd68e 2024-04-14T11:17:43,100 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/14a1ab5a9a4e436b9e8f9921f19bd68e, entries=150, sequenceid=79, filesize=30.2 K 2024-04-14T11:17:43,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/68b291210f4649bf84fc841400f54765 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/68b291210f4649bf84fc841400f54765 2024-04-14T11:17:43,109 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/68b291210f4649bf84fc841400f54765, entries=150, sequenceid=79, filesize=11.7 K 2024-04-14T11:17:43,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/bb113cde374c4e4bac4d8aee8895fa62 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/bb113cde374c4e4bac4d8aee8895fa62 2024-04-14T11:17:43,116 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/bb113cde374c4e4bac4d8aee8895fa62, entries=150, sequenceid=79, filesize=11.7 K 2024-04-14T11:17:43,118 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3035): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 210ms, sequenceid=79, compaction requested=true 2024-04-14T11:17:43,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:43,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:43,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=175 2024-04-14T11:17:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=175 2024-04-14T11:17:43,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:43,122 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:43,122 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=175, resume processing ppid=174 2024-04-14T11:17:43,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-04-14T11:17:43,122 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=175, ppid=174, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 829 msec 2024-04-14T11:17:43,124 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=174, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 840 msec 2024-04-14T11:17:43,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041457d46b459be54d9e825f99300ddfe55c_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093463011/Put/seqid=0 2024-04-14T11:17:43,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093523147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093523147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093523147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093523148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093523148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093523252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093523252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093523253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093523253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093523253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=174 2024-04-14T11:17:43,395 INFO [Thread-2769 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 174 completed 2024-04-14T11:17:43,396 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/8a22e2b9515b42d1b89fbdeeae1ae3bb as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/8a22e2b9515b42d1b89fbdeeae1ae3bb 2024-04-14T11:17:43,397 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees 2024-04-14T11:17:43,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=176 2024-04-14T11:17:43,402 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:43,403 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=176, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:43,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:43,405 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/B of d2b4a72b9f8ab01eabcbb9f313e8e868 into 8a22e2b9515b42d1b89fbdeeae1ae3bb(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:43,405 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:43,405 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/B, priority=13, startTime=1713093462906; duration=0sec 2024-04-14T11:17:43,405 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:43,405 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:B 2024-04-14T11:17:43,405 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:17:43,408 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47936 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:17:43,408 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/C is initiating minor compaction (all files) 2024-04-14T11:17:43,408 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/C in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:43,408 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/7cf5af8c3dfd455fb98f7f6254953398, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/d0ffcaa00bf04b03ad76de7fcc02197f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/3744c3bede2f4d16a3624a2c0de97469, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/bb113cde374c4e4bac4d8aee8895fa62] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=46.8 K 2024-04-14T11:17:43,410 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 7cf5af8c3dfd455fb98f7f6254953398, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1713093461683 2024-04-14T11:17:43,411 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting d0ffcaa00bf04b03ad76de7fcc02197f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1713093461707 2024-04-14T11:17:43,411 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 3744c3bede2f4d16a3624a2c0de97469, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1713093462026 2024-04-14T11:17:43,412 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting bb113cde374c4e4bac4d8aee8895fa62, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1713093462376 2024-04-14T11:17:43,415 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/1b0698db581140c185db13119148363a as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1b0698db581140c185db13119148363a 2024-04-14T11:17:43,425 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/A of d2b4a72b9f8ab01eabcbb9f313e8e868 into 1b0698db581140c185db13119148363a(size=30.3 K), total size for store is 60.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:43,425 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:43,425 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/A, priority=13, startTime=1713093462906; duration=0sec 2024-04-14T11:17:43,425 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:43,425 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:A 2024-04-14T11:17:43,430 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#C#compaction#562 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:43,431 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/41669d25a0124a90a3f610e066945122 is 50, key is test_row_0/C:col10/1713093462376/Put/seqid=0 2024-04-14T11:17:43,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093523455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093523456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093523457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093523458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,461 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/41669d25a0124a90a3f610e066945122 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/41669d25a0124a90a3f610e066945122 2024-04-14T11:17:43,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093523458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,476 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/C of d2b4a72b9f8ab01eabcbb9f313e8e868 into 41669d25a0124a90a3f610e066945122(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:43,476 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:43,476 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/C, priority=12, startTime=1713093462906; duration=0sec 2024-04-14T11:17:43,476 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:43,476 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:C 2024-04-14T11:17:43,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=176 2024-04-14T11:17:43,545 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041457d46b459be54d9e825f99300ddfe55c_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041457d46b459be54d9e825f99300ddfe55c_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:43,547 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/c47f044041e740cca2539c9de1b45f85, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:43,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/c47f044041e740cca2539c9de1b45f85 is 175, key is test_row_0/A:col10/1713093463011/Put/seqid=0 2024-04-14T11:17:43,556 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-04-14T11:17:43,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:43,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:43,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:43,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(53): pid=177 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:43,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:43,558 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/c47f044041e740cca2539c9de1b45f85 2024-04-14T11:17:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:43,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/806daf5737674dd296bd7968bd722a6a is 50, key is test_row_0/B:col10/1713093463011/Put/seqid=0 2024-04-14T11:17:43,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/806daf5737674dd296bd7968bd722a6a 2024-04-14T11:17:43,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/c85b40eca80f4d94b85a55d9e851ad93 is 50, key is test_row_0/C:col10/1713093463011/Put/seqid=0 2024-04-14T11:17:43,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/c85b40eca80f4d94b85a55d9e851ad93 2024-04-14T11:17:43,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/c47f044041e740cca2539c9de1b45f85 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c47f044041e740cca2539c9de1b45f85 2024-04-14T11:17:43,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c47f044041e740cca2539c9de1b45f85, entries=150, sequenceid=95, filesize=30.2 K 2024-04-14T11:17:43,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/806daf5737674dd296bd7968bd722a6a as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/806daf5737674dd296bd7968bd722a6a 2024-04-14T11:17:43,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/806daf5737674dd296bd7968bd722a6a, entries=150, sequenceid=95, filesize=11.7 K 2024-04-14T11:17:43,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/c85b40eca80f4d94b85a55d9e851ad93 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c85b40eca80f4d94b85a55d9e851ad93 2024-04-14T11:17:43,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c85b40eca80f4d94b85a55d9e851ad93, entries=150, sequenceid=95, filesize=11.7 K 2024-04-14T11:17:43,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 553ms, sequenceid=95, compaction requested=true 2024-04-14T11:17:43,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:43,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:43,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:43,675 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:43,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:43,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:43,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:43,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:43,675 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:43,677 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92917 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:43,677 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/A is initiating minor compaction (all files) 2024-04-14T11:17:43,678 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/A in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:43,678 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1b0698db581140c185db13119148363a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/14a1ab5a9a4e436b9e8f9921f19bd68e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c47f044041e740cca2539c9de1b45f85] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=90.7 K 2024-04-14T11:17:43,678 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:43,678 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1b0698db581140c185db13119148363a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/14a1ab5a9a4e436b9e8f9921f19bd68e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c47f044041e740cca2539c9de1b45f85] 2024-04-14T11:17:43,678 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1b0698db581140c185db13119148363a, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1713093462026 2024-04-14T11:17:43,679 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36055 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:43,679 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/B is initiating minor compaction (all files) 2024-04-14T11:17:43,679 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/B in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:43,679 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/8a22e2b9515b42d1b89fbdeeae1ae3bb, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/68b291210f4649bf84fc841400f54765, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/806daf5737674dd296bd7968bd722a6a] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=35.2 K 2024-04-14T11:17:43,679 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 14a1ab5a9a4e436b9e8f9921f19bd68e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1713093462376 2024-04-14T11:17:43,680 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 8a22e2b9515b42d1b89fbdeeae1ae3bb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1713093462026 2024-04-14T11:17:43,681 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 68b291210f4649bf84fc841400f54765, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1713093462376 2024-04-14T11:17:43,681 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c47f044041e740cca2539c9de1b45f85, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1713093463011 2024-04-14T11:17:43,682 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 806daf5737674dd296bd7968bd722a6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1713093463011 2024-04-14T11:17:43,698 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#B#compaction#565 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:43,698 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/be50c5bebecc4e34b25e34aa10224a1d is 50, key is test_row_0/B:col10/1713093463011/Put/seqid=0 2024-04-14T11:17:43,699 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:43,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=176 2024-04-14T11:17:43,705 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20240414acc4c847c54245baa1f464f18d0f88ac_d2b4a72b9f8ab01eabcbb9f313e8e868 store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:43,707 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20240414acc4c847c54245baa1f464f18d0f88ac_d2b4a72b9f8ab01eabcbb9f313e8e868, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:43,707 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414acc4c847c54245baa1f464f18d0f88ac_d2b4a72b9f8ab01eabcbb9f313e8e868 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:43,709 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-04-14T11:17:43,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:43,710 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-04-14T11:17:43,738 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/be50c5bebecc4e34b25e34aa10224a1d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/be50c5bebecc4e34b25e34aa10224a1d 2024-04-14T11:17:43,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414fff4979463204557a5950445f2a5ee0c_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093463146/Put/seqid=0 2024-04-14T11:17:43,747 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/B of d2b4a72b9f8ab01eabcbb9f313e8e868 into be50c5bebecc4e34b25e34aa10224a1d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:43,747 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:43,747 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/B, priority=13, startTime=1713093463675; duration=0sec 2024-04-14T11:17:43,747 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:43,747 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:B 2024-04-14T11:17:43,747 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-04-14T11:17:43,749 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-04-14T11:17:43,749 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-04-14T11:17:43,749 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. because compaction request was cancelled 2024-04-14T11:17:43,749 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:C 2024-04-14T11:17:43,754 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#A#compaction#566 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:43,754 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/fcdd68760320412dbec96d1f8632eea7 is 175, key is test_row_0/A:col10/1713093463011/Put/seqid=0 2024-04-14T11:17:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:43,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:43,775 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414fff4979463204557a5950445f2a5ee0c_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414fff4979463204557a5950445f2a5ee0c_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:43,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/68b045588af44e259f1d3504f94ec95d, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:43,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/68b045588af44e259f1d3504f94ec95d is 175, key is test_row_0/A:col10/1713093463146/Put/seqid=0 2024-04-14T11:17:43,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093523775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093523775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093523777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093523778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093523779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,783 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/fcdd68760320412dbec96d1f8632eea7 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/fcdd68760320412dbec96d1f8632eea7 2024-04-14T11:17:43,791 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/A of d2b4a72b9f8ab01eabcbb9f313e8e868 into fcdd68760320412dbec96d1f8632eea7(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:43,791 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:43,791 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/A, priority=13, startTime=1713093463675; duration=0sec 2024-04-14T11:17:43,791 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:43,791 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:A 2024-04-14T11:17:43,794 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/68b045588af44e259f1d3504f94ec95d 2024-04-14T11:17:43,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/15091b2e5eec4a8396ae9eff79fa0395 is 50, key is test_row_0/B:col10/1713093463146/Put/seqid=0 2024-04-14T11:17:43,854 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/15091b2e5eec4a8396ae9eff79fa0395 2024-04-14T11:17:43,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/9bfd888a99824005bc547f5ccd7533b3 is 50, key is test_row_0/C:col10/1713093463146/Put/seqid=0 2024-04-14T11:17:43,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093523882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093523882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093523882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093523883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093523884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:43,896 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/9bfd888a99824005bc547f5ccd7533b3 2024-04-14T11:17:43,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/68b045588af44e259f1d3504f94ec95d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/68b045588af44e259f1d3504f94ec95d 2024-04-14T11:17:43,910 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/68b045588af44e259f1d3504f94ec95d, entries=150, sequenceid=119, filesize=30.2 K 2024-04-14T11:17:43,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/15091b2e5eec4a8396ae9eff79fa0395 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/15091b2e5eec4a8396ae9eff79fa0395 2024-04-14T11:17:43,920 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/15091b2e5eec4a8396ae9eff79fa0395, entries=150, sequenceid=119, filesize=11.7 K 2024-04-14T11:17:43,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/9bfd888a99824005bc547f5ccd7533b3 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/9bfd888a99824005bc547f5ccd7533b3 2024-04-14T11:17:43,932 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/9bfd888a99824005bc547f5ccd7533b3, entries=150, sequenceid=119, filesize=11.7 K 2024-04-14T11:17:43,933 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(3035): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 223ms, sequenceid=119, compaction requested=true 2024-04-14T11:17:43,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:43,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:43,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=177 2024-04-14T11:17:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=177 2024-04-14T11:17:43,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=177, resume processing ppid=176 2024-04-14T11:17:43,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=177, ppid=176, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 531 msec 2024-04-14T11:17:43,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=176, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees in 539 msec 2024-04-14T11:17:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=176 2024-04-14T11:17:44,006 INFO [Thread-2769 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 176 completed 2024-04-14T11:17:44,008 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees 2024-04-14T11:17:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=178 2024-04-14T11:17:44,012 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=178, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:44,014 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=178, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:44,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:44,088 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:44,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-04-14T11:17:44,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414d509c4e8a8f34e548ef0689ef6de8c77_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093464087/Put/seqid=0 2024-04-14T11:17:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=178 2024-04-14T11:17:44,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093524117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093524118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093524120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,125 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414d509c4e8a8f34e548ef0689ef6de8c77_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414d509c4e8a8f34e548ef0689ef6de8c77_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:44,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093524122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093524126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,127 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/3e8804708dd147f1b78873a0935f8b5e, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:44,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/3e8804708dd147f1b78873a0935f8b5e is 175, key is test_row_0/A:col10/1713093464087/Put/seqid=0 2024-04-14T11:17:44,132 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/3e8804708dd147f1b78873a0935f8b5e 2024-04-14T11:17:44,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/0db9bc7aab404d78b18880f2eefb21b5 is 50, key is test_row_0/B:col10/1713093464087/Put/seqid=0 2024-04-14T11:17:44,167 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-04-14T11:17:44,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:44,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:44,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:44,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] handler.RSProcedureHandler(53): pid=179 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:44,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=179 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=179 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:44,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/0db9bc7aab404d78b18880f2eefb21b5 2024-04-14T11:17:44,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/0bd04cd553fb469da0145d45e1e67480 is 50, key is test_row_0/C:col10/1713093464087/Put/seqid=0 2024-04-14T11:17:44,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/0bd04cd553fb469da0145d45e1e67480 2024-04-14T11:17:44,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/3e8804708dd147f1b78873a0935f8b5e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/3e8804708dd147f1b78873a0935f8b5e 2024-04-14T11:17:44,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/3e8804708dd147f1b78873a0935f8b5e, entries=200, sequenceid=139, filesize=38.8 K 2024-04-14T11:17:44,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/0db9bc7aab404d78b18880f2eefb21b5 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/0db9bc7aab404d78b18880f2eefb21b5 2024-04-14T11:17:44,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/0db9bc7aab404d78b18880f2eefb21b5, entries=150, sequenceid=139, filesize=11.8 K 2024-04-14T11:17:44,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/0bd04cd553fb469da0145d45e1e67480 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/0bd04cd553fb469da0145d45e1e67480 2024-04-14T11:17:44,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093524223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093524225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093524226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093524227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093524227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/0bd04cd553fb469da0145d45e1e67480, entries=150, sequenceid=139, filesize=11.8 K 2024-04-14T11:17:44,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 144ms, sequenceid=139, compaction requested=true 2024-04-14T11:17:44,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:44,232 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:44,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:44,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:44,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:44,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:44,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:44,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:44,233 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:44,234 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101814 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:44,234 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/A is initiating minor compaction (all files) 2024-04-14T11:17:44,234 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/A in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:44,234 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/fcdd68760320412dbec96d1f8632eea7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/68b045588af44e259f1d3504f94ec95d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/3e8804708dd147f1b78873a0935f8b5e] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=99.4 K 2024-04-14T11:17:44,234 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:44,234 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/fcdd68760320412dbec96d1f8632eea7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/68b045588af44e259f1d3504f94ec95d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/3e8804708dd147f1b78873a0935f8b5e] 2024-04-14T11:17:44,235 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting fcdd68760320412dbec96d1f8632eea7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1713093463011 2024-04-14T11:17:44,235 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36308 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:44,235 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/B is initiating minor compaction (all files) 2024-04-14T11:17:44,236 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/B in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:44,236 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 68b045588af44e259f1d3504f94ec95d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1713093463141 2024-04-14T11:17:44,236 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/be50c5bebecc4e34b25e34aa10224a1d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/15091b2e5eec4a8396ae9eff79fa0395, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/0db9bc7aab404d78b18880f2eefb21b5] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=35.5 K 2024-04-14T11:17:44,236 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 3e8804708dd147f1b78873a0935f8b5e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1713093463777 2024-04-14T11:17:44,236 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting be50c5bebecc4e34b25e34aa10224a1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1713093463011 2024-04-14T11:17:44,237 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 15091b2e5eec4a8396ae9eff79fa0395, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1713093463141 2024-04-14T11:17:44,238 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 0db9bc7aab404d78b18880f2eefb21b5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1713093464086 2024-04-14T11:17:44,245 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:44,255 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20240414109b137eca444a23bd022c34b0008fb7_d2b4a72b9f8ab01eabcbb9f313e8e868 store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:44,255 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#B#compaction#574 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:44,256 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/5bceb60a40774c5986ce6bb00a06b14a is 50, key is test_row_0/B:col10/1713093464087/Put/seqid=0 2024-04-14T11:17:44,256 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20240414109b137eca444a23bd022c34b0008fb7_d2b4a72b9f8ab01eabcbb9f313e8e868, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:44,257 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414109b137eca444a23bd022c34b0008fb7_d2b4a72b9f8ab01eabcbb9f313e8e868 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:44,277 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#A#compaction#573 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:44,278 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/25470faa926844f993d3ecec6f4b1d73 is 175, key is test_row_0/A:col10/1713093464087/Put/seqid=0 2024-04-14T11:17:44,283 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/5bceb60a40774c5986ce6bb00a06b14a as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/5bceb60a40774c5986ce6bb00a06b14a 2024-04-14T11:17:44,292 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/B of d2b4a72b9f8ab01eabcbb9f313e8e868 into 5bceb60a40774c5986ce6bb00a06b14a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:44,292 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:44,292 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/B, priority=13, startTime=1713093464233; duration=0sec 2024-04-14T11:17:44,292 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:44,292 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:B 2024-04-14T11:17:44,292 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-04-14T11:17:44,294 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48224 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-04-14T11:17:44,295 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/C is initiating minor compaction (all files) 2024-04-14T11:17:44,295 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/C in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:44,295 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/41669d25a0124a90a3f610e066945122, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c85b40eca80f4d94b85a55d9e851ad93, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/9bfd888a99824005bc547f5ccd7533b3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/0bd04cd553fb469da0145d45e1e67480] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=47.1 K 2024-04-14T11:17:44,295 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 41669d25a0124a90a3f610e066945122, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1713093462376 2024-04-14T11:17:44,296 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting c85b40eca80f4d94b85a55d9e851ad93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1713093463011 2024-04-14T11:17:44,296 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 9bfd888a99824005bc547f5ccd7533b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1713093463141 2024-04-14T11:17:44,297 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 0bd04cd553fb469da0145d45e1e67480, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1713093464086 2024-04-14T11:17:44,307 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#C#compaction#575 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:44,308 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/6ae13013924b459a82732092d039183c is 50, key is test_row_0/C:col10/1713093464087/Put/seqid=0 2024-04-14T11:17:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=178 2024-04-14T11:17:44,320 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=179 2024-04-14T11:17:44,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:44,321 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-04-14T11:17:44,322 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/6ae13013924b459a82732092d039183c as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/6ae13013924b459a82732092d039183c 2024-04-14T11:17:44,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414bc0bb01f99dd44f3a497dabd02d43d12_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093464120/Put/seqid=0 2024-04-14T11:17:44,334 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 4 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/C of d2b4a72b9f8ab01eabcbb9f313e8e868 into 6ae13013924b459a82732092d039183c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:44,334 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:44,334 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/C, priority=12, startTime=1713093464233; duration=0sec 2024-04-14T11:17:44,334 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:44,334 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:C 2024-04-14T11:17:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:44,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:44,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093524446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093524450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093524451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093524450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093524452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093524553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093524556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093524556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093524556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093524556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=178 2024-04-14T11:17:44,693 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/25470faa926844f993d3ecec6f4b1d73 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/25470faa926844f993d3ecec6f4b1d73 2024-04-14T11:17:44,701 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/A of d2b4a72b9f8ab01eabcbb9f313e8e868 into 25470faa926844f993d3ecec6f4b1d73(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:44,701 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:44,701 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/A, priority=13, startTime=1713093464232; duration=0sec 2024-04-14T11:17:44,701 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:44,701 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:A 2024-04-14T11:17:44,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093524757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093524759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093524759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093524760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,762 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414bc0bb01f99dd44f3a497dabd02d43d12_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414bc0bb01f99dd44f3a497dabd02d43d12_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:44,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093524761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:44,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/d6f8df5a108f4589aaf871e2bb7a3c35, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:44,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/d6f8df5a108f4589aaf871e2bb7a3c35 is 175, key is test_row_0/A:col10/1713093464120/Put/seqid=0 2024-04-14T11:17:44,772 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/d6f8df5a108f4589aaf871e2bb7a3c35 2024-04-14T11:17:44,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/a8e02afdc3f54d449b19522089bede82 is 50, key is test_row_0/B:col10/1713093464120/Put/seqid=0 2024-04-14T11:17:44,801 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/a8e02afdc3f54d449b19522089bede82 2024-04-14T11:17:44,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/49baa07997d74ca7a248ad60c15a1317 is 50, key is test_row_0/C:col10/1713093464120/Put/seqid=0 2024-04-14T11:17:44,839 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/49baa07997d74ca7a248ad60c15a1317 2024-04-14T11:17:44,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/d6f8df5a108f4589aaf871e2bb7a3c35 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/d6f8df5a108f4589aaf871e2bb7a3c35 2024-04-14T11:17:44,853 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/d6f8df5a108f4589aaf871e2bb7a3c35, entries=150, sequenceid=159, filesize=30.4 K 2024-04-14T11:17:44,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/a8e02afdc3f54d449b19522089bede82 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/a8e02afdc3f54d449b19522089bede82 2024-04-14T11:17:44,859 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/a8e02afdc3f54d449b19522089bede82, entries=150, sequenceid=159, filesize=11.8 K 2024-04-14T11:17:44,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/49baa07997d74ca7a248ad60c15a1317 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/49baa07997d74ca7a248ad60c15a1317 2024-04-14T11:17:44,866 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/49baa07997d74ca7a248ad60c15a1317, entries=150, sequenceid=159, filesize=11.8 K 2024-04-14T11:17:44,867 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(3035): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 546ms, sequenceid=159, compaction requested=false 2024-04-14T11:17:44,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:44,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:44,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=179}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=179 2024-04-14T11:17:44,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=179 2024-04-14T11:17:44,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=179, resume processing ppid=178 2024-04-14T11:17:44,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=179, ppid=178, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 854 msec 2024-04-14T11:17:44,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=178, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=178, table=TestAcidGuarantees in 863 msec 2024-04-14T11:17:45,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:45,060 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:45,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-04-14T11:17:45,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404147553171a61f64968b8a8bfb30fa27a70_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093465059/Put/seqid=0 2024-04-14T11:17:45,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093525073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093525073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,077 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404147553171a61f64968b8a8bfb30fa27a70_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404147553171a61f64968b8a8bfb30fa27a70_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:45,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093525075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,078 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/fb6737a844d24fe3a62884f3b5f6768d, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:45,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093525077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/fb6737a844d24fe3a62884f3b5f6768d is 175, key is test_row_0/A:col10/1713093465059/Put/seqid=0 2024-04-14T11:17:45,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093525077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,084 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/fb6737a844d24fe3a62884f3b5f6768d 2024-04-14T11:17:45,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/e3b0d2f09b484b4b8141d88dca3a8f54 is 50, key is test_row_0/B:col10/1713093465059/Put/seqid=0 2024-04-14T11:17:45,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=178 2024-04-14T11:17:45,117 INFO [Thread-2769 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 178 completed 2024-04-14T11:17:45,118 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=180, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees 2024-04-14T11:17:45,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=180 2024-04-14T11:17:45,121 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=180, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:45,122 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=180, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:45,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=181, ppid=180, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:45,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093525178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093525178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093525179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093525180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093525180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=180 2024-04-14T11:17:45,274 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-04-14T11:17:45,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:45,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:45,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:45,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(53): pid=181 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:45,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:45,341 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-04-14T11:17:45,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093525381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093525381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093525383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093525383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093525384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=180 2024-04-14T11:17:45,427 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-04-14T11:17:45,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:45,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:45,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:45,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] handler.RSProcedureHandler(53): pid=181 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:45,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=181 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:45,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=181 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:45,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/e3b0d2f09b484b4b8141d88dca3a8f54 2024-04-14T11:17:45,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/1cb92373e85a481aa7f165d7a343d7e5 is 50, key is test_row_0/C:col10/1713093465059/Put/seqid=0 2024-04-14T11:17:45,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/1cb92373e85a481aa7f165d7a343d7e5 2024-04-14T11:17:45,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/fb6737a844d24fe3a62884f3b5f6768d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/fb6737a844d24fe3a62884f3b5f6768d 2024-04-14T11:17:45,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/fb6737a844d24fe3a62884f3b5f6768d, entries=200, sequenceid=180, filesize=38.8 K 2024-04-14T11:17:45,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/e3b0d2f09b484b4b8141d88dca3a8f54 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/e3b0d2f09b484b4b8141d88dca3a8f54 2024-04-14T11:17:45,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/e3b0d2f09b484b4b8141d88dca3a8f54, entries=150, sequenceid=180, filesize=11.8 K 2024-04-14T11:17:45,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/1cb92373e85a481aa7f165d7a343d7e5 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/1cb92373e85a481aa7f165d7a343d7e5 2024-04-14T11:17:45,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/1cb92373e85a481aa7f165d7a343d7e5, entries=150, sequenceid=180, filesize=11.8 K 2024-04-14T11:17:45,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 476ms, sequenceid=180, compaction requested=true 2024-04-14T11:17:45,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:45,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:45,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:45,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:45,536 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:45,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:45,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:45,536 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:45,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:45,538 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36710 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:45,538 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102216 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:45,538 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/B is initiating minor compaction (all files) 2024-04-14T11:17:45,538 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/A is initiating minor compaction (all files) 2024-04-14T11:17:45,538 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/B in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:45,538 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/A in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:45,538 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/25470faa926844f993d3ecec6f4b1d73, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/d6f8df5a108f4589aaf871e2bb7a3c35, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/fb6737a844d24fe3a62884f3b5f6768d] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=99.8 K 2024-04-14T11:17:45,538 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/5bceb60a40774c5986ce6bb00a06b14a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/a8e02afdc3f54d449b19522089bede82, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/e3b0d2f09b484b4b8141d88dca3a8f54] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=35.8 K 2024-04-14T11:17:45,538 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:45,538 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/25470faa926844f993d3ecec6f4b1d73, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/d6f8df5a108f4589aaf871e2bb7a3c35, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/fb6737a844d24fe3a62884f3b5f6768d] 2024-04-14T11:17:45,539 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 5bceb60a40774c5986ce6bb00a06b14a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1713093464086 2024-04-14T11:17:45,539 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 25470faa926844f993d3ecec6f4b1d73, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1713093464086 2024-04-14T11:17:45,540 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting a8e02afdc3f54d449b19522089bede82, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1713093464113 2024-04-14T11:17:45,540 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting d6f8df5a108f4589aaf871e2bb7a3c35, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1713093464113 2024-04-14T11:17:45,541 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting e3b0d2f09b484b4b8141d88dca3a8f54, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1713093464435 2024-04-14T11:17:45,541 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting fb6737a844d24fe3a62884f3b5f6768d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1713093464435 2024-04-14T11:17:45,548 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:45,548 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#B#compaction#582 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:45,549 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/87195e87b9f04f4d8a91f6a2bd774a65 is 50, key is test_row_0/B:col10/1713093465059/Put/seqid=0 2024-04-14T11:17:45,549 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202404147ed8802088c74c2fb5e7917f689d88b7_d2b4a72b9f8ab01eabcbb9f313e8e868 store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:45,551 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202404147ed8802088c74c2fb5e7917f689d88b7_d2b4a72b9f8ab01eabcbb9f313e8e868, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:45,551 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404147ed8802088c74c2fb5e7917f689d88b7_d2b4a72b9f8ab01eabcbb9f313e8e868 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:45,556 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#A#compaction#583 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:45,557 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/442b47066f1d42d2a8441a9192ab1da4 is 175, key is test_row_0/A:col10/1713093465059/Put/seqid=0 2024-04-14T11:17:45,561 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/87195e87b9f04f4d8a91f6a2bd774a65 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/87195e87b9f04f4d8a91f6a2bd774a65 2024-04-14T11:17:45,570 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/B of d2b4a72b9f8ab01eabcbb9f313e8e868 into 87195e87b9f04f4d8a91f6a2bd774a65(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:45,570 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:45,570 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/B, priority=13, startTime=1713093465536; duration=0sec 2024-04-14T11:17:45,570 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:45,570 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:B 2024-04-14T11:17:45,570 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:45,571 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/442b47066f1d42d2a8441a9192ab1da4 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/442b47066f1d42d2a8441a9192ab1da4 2024-04-14T11:17:45,572 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36676 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:45,572 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/C is initiating minor compaction (all files) 2024-04-14T11:17:45,572 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/C in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:45,572 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/6ae13013924b459a82732092d039183c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/49baa07997d74ca7a248ad60c15a1317, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/1cb92373e85a481aa7f165d7a343d7e5] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=35.8 K 2024-04-14T11:17:45,572 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 6ae13013924b459a82732092d039183c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1713093464086 2024-04-14T11:17:45,573 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 49baa07997d74ca7a248ad60c15a1317, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1713093464113 2024-04-14T11:17:45,573 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 1cb92373e85a481aa7f165d7a343d7e5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1713093464435 2024-04-14T11:17:45,578 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/A of d2b4a72b9f8ab01eabcbb9f313e8e868 into 442b47066f1d42d2a8441a9192ab1da4(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:45,578 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:45,578 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/A, priority=13, startTime=1713093465536; duration=0sec 2024-04-14T11:17:45,578 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:45,578 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:A 2024-04-14T11:17:45,579 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,580 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#C#compaction#584 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:45,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=181 2024-04-14T11:17:45,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:45,580 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-04-14T11:17:45,580 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/3b27f0036a154f748343c496932d7cf3 is 50, key is test_row_0/C:col10/1713093465059/Put/seqid=0 2024-04-14T11:17:45,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041401ac02d3670d4c85bbc96cd2134ff58c_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093465072/Put/seqid=0 2024-04-14T11:17:45,601 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/3b27f0036a154f748343c496932d7cf3 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/3b27f0036a154f748343c496932d7cf3 2024-04-14T11:17:45,608 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/C of d2b4a72b9f8ab01eabcbb9f313e8e868 into 3b27f0036a154f748343c496932d7cf3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:45,608 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:45,608 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/C, priority=13, startTime=1713093465536; duration=0sec 2024-04-14T11:17:45,608 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:45,608 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:C 2024-04-14T11:17:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:45,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:45,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093525695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093525695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093525696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093525696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093525697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=180 2024-04-14T11:17:45,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093525799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093525799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093525800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093525800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093525802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:45,995 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041401ac02d3670d4c85bbc96cd2134ff58c_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041401ac02d3670d4c85bbc96cd2134ff58c_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:45,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/b471d7a401884b3fbebbeca9c0b1cc44, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:45,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/b471d7a401884b3fbebbeca9c0b1cc44 is 175, key is test_row_0/A:col10/1713093465072/Put/seqid=0 2024-04-14T11:17:46,001 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/b471d7a401884b3fbebbeca9c0b1cc44 2024-04-14T11:17:46,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093526002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093526003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093526003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093526004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093526005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/962a8b751a534bb693a93b885f5416b0 is 50, key is test_row_0/B:col10/1713093465072/Put/seqid=0 2024-04-14T11:17:46,015 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/962a8b751a534bb693a93b885f5416b0 2024-04-14T11:17:46,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/a13d51ae71ef451bac6fb35d79c86f7a is 50, key is test_row_0/C:col10/1713093465072/Put/seqid=0 2024-04-14T11:17:46,027 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/a13d51ae71ef451bac6fb35d79c86f7a 2024-04-14T11:17:46,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/b471d7a401884b3fbebbeca9c0b1cc44 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/b471d7a401884b3fbebbeca9c0b1cc44 2024-04-14T11:17:46,037 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/b471d7a401884b3fbebbeca9c0b1cc44, entries=150, sequenceid=200, filesize=30.4 K 2024-04-14T11:17:46,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/962a8b751a534bb693a93b885f5416b0 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/962a8b751a534bb693a93b885f5416b0 2024-04-14T11:17:46,043 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/962a8b751a534bb693a93b885f5416b0, entries=150, sequenceid=200, filesize=11.8 K 2024-04-14T11:17:46,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/a13d51ae71ef451bac6fb35d79c86f7a as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/a13d51ae71ef451bac6fb35d79c86f7a 2024-04-14T11:17:46,049 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/a13d51ae71ef451bac6fb35d79c86f7a, entries=150, sequenceid=200, filesize=11.8 K 2024-04-14T11:17:46,050 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(3035): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 470ms, sequenceid=200, compaction requested=false 2024-04-14T11:17:46,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:46,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:46,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=181}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=181 2024-04-14T11:17:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=181 2024-04-14T11:17:46,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=181, resume processing ppid=180 2024-04-14T11:17:46,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=181, ppid=180, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 929 msec 2024-04-14T11:17:46,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=180, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=180, table=TestAcidGuarantees in 935 msec 2024-04-14T11:17:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=180 2024-04-14T11:17:46,226 INFO [Thread-2769 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 180 completed 2024-04-14T11:17:46,227 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=182, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees 2024-04-14T11:17:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=182 2024-04-14T11:17:46,230 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=182, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:46,231 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=182, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:46,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=183, ppid=182, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:46,308 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:46,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-04-14T11:17:46,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404149bba818881994c41a20cb15e73d9b1c0_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093465694/Put/seqid=0 2024-04-14T11:17:46,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093526317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093526319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093526319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093526320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093526321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,326 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404149bba818881994c41a20cb15e73d9b1c0_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404149bba818881994c41a20cb15e73d9b1c0_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:46,327 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/30dbdc4f44b04eb5aea9dca61b179924, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:46,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/30dbdc4f44b04eb5aea9dca61b179924 is 175, key is test_row_0/A:col10/1713093465694/Put/seqid=0 2024-04-14T11:17:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=182 2024-04-14T11:17:46,383 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=183 2024-04-14T11:17:46,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:46,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:46,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:46,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] handler.RSProcedureHandler(53): pid=183 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:46,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=183 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=183 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:46,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093526421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093526423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093526423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093526424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093526425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=182 2024-04-14T11:17:46,536 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=183 2024-04-14T11:17:46,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:46,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:46,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:46,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] handler.RSProcedureHandler(53): pid=183 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:46,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=183 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=183 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:46,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093526625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093526625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093526626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093526628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093526629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,689 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=183 2024-04-14T11:17:46,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:46,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:46,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:46,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] handler.RSProcedureHandler(53): pid=183 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:46,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=183 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=183 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:46,733 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/30dbdc4f44b04eb5aea9dca61b179924 2024-04-14T11:17:46,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/f512de4bda1041a585ea1e4837078ddf is 50, key is test_row_0/B:col10/1713093465694/Put/seqid=0 2024-04-14T11:17:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=182 2024-04-14T11:17:46,843 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=183 2024-04-14T11:17:46,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:46,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:46,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:46,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] handler.RSProcedureHandler(53): pid=183 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:46,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=183 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:46,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=183 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:46,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093526927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093526929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093526930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093526932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:46,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093526932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,995 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:46,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=183 2024-04-14T11:17:46,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:46,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:46,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:46,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] handler.RSProcedureHandler(53): pid=183 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:46,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=183 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:46,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=183 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:47,148 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:47,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=183 2024-04-14T11:17:47,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:47,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:47,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:47,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] handler.RSProcedureHandler(53): pid=183 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:47,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=183 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:47,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/f512de4bda1041a585ea1e4837078ddf 2024-04-14T11:17:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=183 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:47,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/4ee7ca1e9746479a81850c8554a595d4 is 50, key is test_row_0/C:col10/1713093465694/Put/seqid=0 2024-04-14T11:17:47,303 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:47,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=183 2024-04-14T11:17:47,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:47,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:47,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:47,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] handler.RSProcedureHandler(53): pid=183 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:47,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=183 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:47,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=183 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:47,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=182 2024-04-14T11:17:47,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093527432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:47,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093527434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:47,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093527435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:47,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093527435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:47,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093527435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:47,455 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:47,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=183 2024-04-14T11:17:47,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:47,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:47,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:47,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] handler.RSProcedureHandler(53): pid=183 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:47,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=183 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=183 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:47,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/4ee7ca1e9746479a81850c8554a595d4 2024-04-14T11:17:47,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/30dbdc4f44b04eb5aea9dca61b179924 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/30dbdc4f44b04eb5aea9dca61b179924 2024-04-14T11:17:47,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/30dbdc4f44b04eb5aea9dca61b179924, entries=200, sequenceid=222, filesize=38.8 K 2024-04-14T11:17:47,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/f512de4bda1041a585ea1e4837078ddf as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/f512de4bda1041a585ea1e4837078ddf 2024-04-14T11:17:47,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/f512de4bda1041a585ea1e4837078ddf, entries=150, sequenceid=222, filesize=11.8 K 2024-04-14T11:17:47,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/4ee7ca1e9746479a81850c8554a595d4 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4ee7ca1e9746479a81850c8554a595d4 2024-04-14T11:17:47,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4ee7ca1e9746479a81850c8554a595d4, entries=150, sequenceid=222, filesize=11.8 K 2024-04-14T11:17:47,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 1278ms, sequenceid=222, compaction requested=true 2024-04-14T11:17:47,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:47,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:47,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:47,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:47,587 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:47,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:47,587 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:47,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:47,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:47,588 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102318 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:47,588 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/A is initiating minor compaction (all files) 2024-04-14T11:17:47,588 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/A in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:47,588 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/442b47066f1d42d2a8441a9192ab1da4, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/b471d7a401884b3fbebbeca9c0b1cc44, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/30dbdc4f44b04eb5aea9dca61b179924] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=99.9 K 2024-04-14T11:17:47,588 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36812 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:47,588 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/B is initiating minor compaction (all files) 2024-04-14T11:17:47,588 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:47,588 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/B in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:47,588 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/442b47066f1d42d2a8441a9192ab1da4, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/b471d7a401884b3fbebbeca9c0b1cc44, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/30dbdc4f44b04eb5aea9dca61b179924] 2024-04-14T11:17:47,588 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/87195e87b9f04f4d8a91f6a2bd774a65, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/962a8b751a534bb693a93b885f5416b0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/f512de4bda1041a585ea1e4837078ddf] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=35.9 K 2024-04-14T11:17:47,589 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 442b47066f1d42d2a8441a9192ab1da4, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1713093464435 2024-04-14T11:17:47,589 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 87195e87b9f04f4d8a91f6a2bd774a65, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1713093464435 2024-04-14T11:17:47,589 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting b471d7a401884b3fbebbeca9c0b1cc44, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1713093465071 2024-04-14T11:17:47,590 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 962a8b751a534bb693a93b885f5416b0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1713093465071 2024-04-14T11:17:47,590 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 30dbdc4f44b04eb5aea9dca61b179924, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1713093465694 2024-04-14T11:17:47,590 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting f512de4bda1041a585ea1e4837078ddf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1713093465694 2024-04-14T11:17:47,597 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:47,598 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#B#compaction#592 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:47,598 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/17ece2d83242494eb2d8b69de47105f6 is 50, key is test_row_0/B:col10/1713093465694/Put/seqid=0 2024-04-14T11:17:47,599 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20240414edcd2385e7b54e3685d6c802026e5403_d2b4a72b9f8ab01eabcbb9f313e8e868 store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:47,601 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20240414edcd2385e7b54e3685d6c802026e5403_d2b4a72b9f8ab01eabcbb9f313e8e868, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:47,601 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414edcd2385e7b54e3685d6c802026e5403_d2b4a72b9f8ab01eabcbb9f313e8e868 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:47,608 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#A#compaction#591 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:47,608 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:47,609 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/22ea9944e5bc44b1a8d62c0bfcfe6a80 is 175, key is test_row_0/A:col10/1713093465694/Put/seqid=0 2024-04-14T11:17:47,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=183 2024-04-14T11:17:47,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:47,609 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-04-14T11:17:47,613 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/17ece2d83242494eb2d8b69de47105f6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/17ece2d83242494eb2d8b69de47105f6 2024-04-14T11:17:47,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041454d48dd67cbd42c0a920cb1e93dbae82_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093466319/Put/seqid=0 2024-04-14T11:17:47,621 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/B of d2b4a72b9f8ab01eabcbb9f313e8e868 into 17ece2d83242494eb2d8b69de47105f6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:47,621 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:47,621 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/B, priority=13, startTime=1713093467587; duration=0sec 2024-04-14T11:17:47,621 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:47,621 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:B 2024-04-14T11:17:47,622 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:47,623 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/22ea9944e5bc44b1a8d62c0bfcfe6a80 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/22ea9944e5bc44b1a8d62c0bfcfe6a80 2024-04-14T11:17:47,623 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36778 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:47,624 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/C is initiating minor compaction (all files) 2024-04-14T11:17:47,624 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/C in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:47,624 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/3b27f0036a154f748343c496932d7cf3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/a13d51ae71ef451bac6fb35d79c86f7a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4ee7ca1e9746479a81850c8554a595d4] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=35.9 K 2024-04-14T11:17:47,624 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 3b27f0036a154f748343c496932d7cf3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1713093464435 2024-04-14T11:17:47,625 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting a13d51ae71ef451bac6fb35d79c86f7a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1713093465071 2024-04-14T11:17:47,625 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 4ee7ca1e9746479a81850c8554a595d4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1713093465694 2024-04-14T11:17:47,627 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041454d48dd67cbd42c0a920cb1e93dbae82_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041454d48dd67cbd42c0a920cb1e93dbae82_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:47,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/f24a564099444e8fa29bbcebd8f3daa7, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:47,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/f24a564099444e8fa29bbcebd8f3daa7 is 175, key is test_row_0/A:col10/1713093466319/Put/seqid=0 2024-04-14T11:17:47,630 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/A of d2b4a72b9f8ab01eabcbb9f313e8e868 into 22ea9944e5bc44b1a8d62c0bfcfe6a80(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:47,630 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:47,630 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/A, priority=13, startTime=1713093467586; duration=0sec 2024-04-14T11:17:47,630 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:47,630 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:A 2024-04-14T11:17:47,638 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/f24a564099444e8fa29bbcebd8f3daa7 2024-04-14T11:17:47,638 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#C#compaction#594 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:47,639 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/a4f0d130c8f940beaa45aae9c95a105f is 50, key is test_row_0/C:col10/1713093465694/Put/seqid=0 2024-04-14T11:17:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/bac96139f3ac42b9922c1ee783f2e526 is 50, key is test_row_0/B:col10/1713093466319/Put/seqid=0 2024-04-14T11:17:47,654 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/a4f0d130c8f940beaa45aae9c95a105f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/a4f0d130c8f940beaa45aae9c95a105f 2024-04-14T11:17:47,659 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/C of d2b4a72b9f8ab01eabcbb9f313e8e868 into a4f0d130c8f940beaa45aae9c95a105f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:47,660 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:47,660 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/C, priority=13, startTime=1713093467587; duration=0sec 2024-04-14T11:17:47,660 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:47,660 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:C 2024-04-14T11:17:48,054 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/bac96139f3ac42b9922c1ee783f2e526 2024-04-14T11:17:48,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/e37f1c55399c4ef58606270fe7438679 is 50, key is test_row_0/C:col10/1713093466319/Put/seqid=0 2024-04-14T11:17:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=182 2024-04-14T11:17:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:48,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:48,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093528452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093528453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093528453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093528454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093528454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,467 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/e37f1c55399c4ef58606270fe7438679 2024-04-14T11:17:48,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/f24a564099444e8fa29bbcebd8f3daa7 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/f24a564099444e8fa29bbcebd8f3daa7 2024-04-14T11:17:48,478 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/f24a564099444e8fa29bbcebd8f3daa7, entries=150, sequenceid=238, filesize=30.4 K 2024-04-14T11:17:48,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/bac96139f3ac42b9922c1ee783f2e526 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/bac96139f3ac42b9922c1ee783f2e526 2024-04-14T11:17:48,484 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/bac96139f3ac42b9922c1ee783f2e526, entries=150, sequenceid=238, filesize=11.8 K 2024-04-14T11:17:48,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/e37f1c55399c4ef58606270fe7438679 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/e37f1c55399c4ef58606270fe7438679 2024-04-14T11:17:48,489 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/e37f1c55399c4ef58606270fe7438679, entries=150, sequenceid=238, filesize=11.8 K 2024-04-14T11:17:48,490 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(3035): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 881ms, sequenceid=238, compaction requested=false 2024-04-14T11:17:48,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:48,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:48,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=183}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=183 2024-04-14T11:17:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=183 2024-04-14T11:17:48,493 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=183, resume processing ppid=182 2024-04-14T11:17:48,493 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=183, ppid=182, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2600 sec 2024-04-14T11:17:48,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=182, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=182, table=TestAcidGuarantees in 2.2670 sec 2024-04-14T11:17:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:48,558 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:48,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-04-14T11:17:48,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404142c0f22d2a0ce448bb519165c0c5961da_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093468453/Put/seqid=0 2024-04-14T11:17:48,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093528569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093528571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093528571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093528572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093528572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,577 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404142c0f22d2a0ce448bb519165c0c5961da_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404142c0f22d2a0ce448bb519165c0c5961da_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:48,578 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/ff87bb17db764bec89decb8583f1f2b1, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:48,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/ff87bb17db764bec89decb8583f1f2b1 is 175, key is test_row_0/A:col10/1713093468453/Put/seqid=0 2024-04-14T11:17:48,584 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/ff87bb17db764bec89decb8583f1f2b1 2024-04-14T11:17:48,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/103b144b0bf94b3c871ee1101bc48817 is 50, key is test_row_0/B:col10/1713093468453/Put/seqid=0 2024-04-14T11:17:48,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093528673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093528675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093528675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093528676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093528676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093528877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093528878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093528878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093528879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093528879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:48,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/103b144b0bf94b3c871ee1101bc48817 2024-04-14T11:17:49,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/e70ca28462ae4934a034ad17aeca7d0d is 50, key is test_row_0/C:col10/1713093468453/Put/seqid=0 2024-04-14T11:17:49,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/e70ca28462ae4934a034ad17aeca7d0d 2024-04-14T11:17:49,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/ff87bb17db764bec89decb8583f1f2b1 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ff87bb17db764bec89decb8583f1f2b1 2024-04-14T11:17:49,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ff87bb17db764bec89decb8583f1f2b1, entries=150, sequenceid=264, filesize=30.5 K 2024-04-14T11:17:49,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/103b144b0bf94b3c871ee1101bc48817 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/103b144b0bf94b3c871ee1101bc48817 2024-04-14T11:17:49,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/103b144b0bf94b3c871ee1101bc48817, entries=150, sequenceid=264, filesize=12.0 K 2024-04-14T11:17:49,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/e70ca28462ae4934a034ad17aeca7d0d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/e70ca28462ae4934a034ad17aeca7d0d 2024-04-14T11:17:49,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/e70ca28462ae4934a034ad17aeca7d0d, entries=150, sequenceid=264, filesize=12.0 K 2024-04-14T11:17:49,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=80.51 KB/82440 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 474ms, sequenceid=264, compaction requested=true 2024-04-14T11:17:49,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:49,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:49,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:49,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:49,032 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:49,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:49,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:49,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:49,032 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:49,033 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93926 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:49,033 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/A is initiating minor compaction (all files) 2024-04-14T11:17:49,033 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/A in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:49,033 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37064 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:49,033 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/22ea9944e5bc44b1a8d62c0bfcfe6a80, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/f24a564099444e8fa29bbcebd8f3daa7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ff87bb17db764bec89decb8583f1f2b1] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=91.7 K 2024-04-14T11:17:49,033 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/B is initiating minor compaction (all files) 2024-04-14T11:17:49,034 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:49,034 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/B in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:49,034 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/22ea9944e5bc44b1a8d62c0bfcfe6a80, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/f24a564099444e8fa29bbcebd8f3daa7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ff87bb17db764bec89decb8583f1f2b1] 2024-04-14T11:17:49,034 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/17ece2d83242494eb2d8b69de47105f6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/bac96139f3ac42b9922c1ee783f2e526, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/103b144b0bf94b3c871ee1101bc48817] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=36.2 K 2024-04-14T11:17:49,034 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 22ea9944e5bc44b1a8d62c0bfcfe6a80, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1713093465694 2024-04-14T11:17:49,034 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 17ece2d83242494eb2d8b69de47105f6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1713093465694 2024-04-14T11:17:49,035 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting f24a564099444e8fa29bbcebd8f3daa7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1713093466316 2024-04-14T11:17:49,035 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting bac96139f3ac42b9922c1ee783f2e526, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1713093466316 2024-04-14T11:17:49,035 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting ff87bb17db764bec89decb8583f1f2b1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1713093468452 2024-04-14T11:17:49,035 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 103b144b0bf94b3c871ee1101bc48817, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1713093468452 2024-04-14T11:17:49,041 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:49,050 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#B#compaction#601 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:49,051 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/124ae69d25f94e0bb566c62c9c9cbe45 is 50, key is test_row_0/B:col10/1713093468453/Put/seqid=0 2024-04-14T11:17:49,055 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202404146b31217ea3df4ff8b3445be775f1573d_d2b4a72b9f8ab01eabcbb9f313e8e868 store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:49,056 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202404146b31217ea3df4ff8b3445be775f1573d_d2b4a72b9f8ab01eabcbb9f313e8e868, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:49,057 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404146b31217ea3df4ff8b3445be775f1573d_d2b4a72b9f8ab01eabcbb9f313e8e868 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:49,068 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/124ae69d25f94e0bb566c62c9c9cbe45 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/124ae69d25f94e0bb566c62c9c9cbe45 2024-04-14T11:17:49,074 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/B of d2b4a72b9f8ab01eabcbb9f313e8e868 into 124ae69d25f94e0bb566c62c9c9cbe45(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:49,074 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:49,074 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/B, priority=13, startTime=1713093469032; duration=0sec 2024-04-14T11:17:49,074 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:49,074 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:B 2024-04-14T11:17:49,074 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:49,076 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37030 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:49,076 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/C is initiating minor compaction (all files) 2024-04-14T11:17:49,076 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/C in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:49,076 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/a4f0d130c8f940beaa45aae9c95a105f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/e37f1c55399c4ef58606270fe7438679, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/e70ca28462ae4934a034ad17aeca7d0d] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=36.2 K 2024-04-14T11:17:49,077 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting a4f0d130c8f940beaa45aae9c95a105f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1713093465694 2024-04-14T11:17:49,077 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting e37f1c55399c4ef58606270fe7438679, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1713093466316 2024-04-14T11:17:49,078 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting e70ca28462ae4934a034ad17aeca7d0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1713093468452 2024-04-14T11:17:49,087 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#C#compaction#602 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:49,087 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/7e6e9fbf44c548afaf73ca2d3988b40f is 50, key is test_row_0/C:col10/1713093468453/Put/seqid=0 2024-04-14T11:17:49,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:49,181 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:49,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-04-14T11:17:49,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041496c808e76b8143508063801cf8753dc8_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093468562/Put/seqid=0 2024-04-14T11:17:49,199 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041496c808e76b8143508063801cf8753dc8_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041496c808e76b8143508063801cf8753dc8_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:49,200 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/26a441709e84433e84e3cf08d7ede4a5, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:49,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093529197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/26a441709e84433e84e3cf08d7ede4a5 is 175, key is test_row_0/A:col10/1713093468562/Put/seqid=0 2024-04-14T11:17:49,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093529199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093529199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093529200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093529201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,206 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/26a441709e84433e84e3cf08d7ede4a5 2024-04-14T11:17:49,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/60bdb660926b4605b0ad195129f852a8 is 50, key is test_row_0/B:col10/1713093468562/Put/seqid=0 2024-04-14T11:17:49,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093529302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093529304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093529304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093529304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093529305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,464 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#A#compaction#600 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:49,464 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/83bec1e9658d4c5cbd2de8d4292a6142 is 175, key is test_row_0/A:col10/1713093468453/Put/seqid=0 2024-04-14T11:17:49,499 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/7e6e9fbf44c548afaf73ca2d3988b40f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/7e6e9fbf44c548afaf73ca2d3988b40f 2024-04-14T11:17:49,506 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/C of d2b4a72b9f8ab01eabcbb9f313e8e868 into 7e6e9fbf44c548afaf73ca2d3988b40f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:49,506 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:49,506 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/C, priority=13, startTime=1713093469032; duration=0sec 2024-04-14T11:17:49,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,506 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:49,506 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:C 2024-04-14T11:17:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093529504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093529506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093529507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093529507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093529508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/60bdb660926b4605b0ad195129f852a8 2024-04-14T11:17:49,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/22a91df9f9cc4b6388de2f7c0b1cca01 is 50, key is test_row_0/C:col10/1713093468562/Put/seqid=0 2024-04-14T11:17:49,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/22a91df9f9cc4b6388de2f7c0b1cca01 2024-04-14T11:17:49,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/26a441709e84433e84e3cf08d7ede4a5 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/26a441709e84433e84e3cf08d7ede4a5 2024-04-14T11:17:49,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/26a441709e84433e84e3cf08d7ede4a5, entries=200, sequenceid=281, filesize=39.0 K 2024-04-14T11:17:49,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/60bdb660926b4605b0ad195129f852a8 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/60bdb660926b4605b0ad195129f852a8 2024-04-14T11:17:49,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/60bdb660926b4605b0ad195129f852a8, entries=150, sequenceid=281, filesize=12.0 K 2024-04-14T11:17:49,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/22a91df9f9cc4b6388de2f7c0b1cca01 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/22a91df9f9cc4b6388de2f7c0b1cca01 2024-04-14T11:17:49,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/22a91df9f9cc4b6388de2f7c0b1cca01, entries=150, sequenceid=281, filesize=12.0 K 2024-04-14T11:17:49,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 502ms, sequenceid=281, compaction requested=false 2024-04-14T11:17:49,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:49,813 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:49,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-04-14T11:17:49,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:49,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404146ef7c45d5de54f699d0f30761279ed6d_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093469200/Put/seqid=0 2024-04-14T11:17:49,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093529826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093529827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093529827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093529828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093529828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,876 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/83bec1e9658d4c5cbd2de8d4292a6142 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/83bec1e9658d4c5cbd2de8d4292a6142 2024-04-14T11:17:49,884 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/A of d2b4a72b9f8ab01eabcbb9f313e8e868 into 83bec1e9658d4c5cbd2de8d4292a6142(size=31.1 K), total size for store is 70.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:49,884 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:49,884 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/A, priority=13, startTime=1713093469032; duration=0sec 2024-04-14T11:17:49,884 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:49,884 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:A 2024-04-14T11:17:49,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093529932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093529933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093529933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093529933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:49,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093529934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093530136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093530136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093530136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093530137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093530137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,245 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404146ef7c45d5de54f699d0f30761279ed6d_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404146ef7c45d5de54f699d0f30761279ed6d_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:50,246 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/35ef2a04a07b49129b92a2903c6a0b08, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:50,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/35ef2a04a07b49129b92a2903c6a0b08 is 175, key is test_row_0/A:col10/1713093469200/Put/seqid=0 2024-04-14T11:17:50,252 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/35ef2a04a07b49129b92a2903c6a0b08 2024-04-14T11:17:50,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/cc8f4306770245bf9aa37f3da253b352 is 50, key is test_row_0/B:col10/1713093469200/Put/seqid=0 2024-04-14T11:17:50,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/cc8f4306770245bf9aa37f3da253b352 2024-04-14T11:17:50,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/4e515c961afb41fd97c9fb9e083606eb is 50, key is test_row_0/C:col10/1713093469200/Put/seqid=0 2024-04-14T11:17:50,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/4e515c961afb41fd97c9fb9e083606eb 2024-04-14T11:17:50,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/35ef2a04a07b49129b92a2903c6a0b08 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/35ef2a04a07b49129b92a2903c6a0b08 2024-04-14T11:17:50,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/35ef2a04a07b49129b92a2903c6a0b08, entries=150, sequenceid=305, filesize=30.5 K 2024-04-14T11:17:50,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/cc8f4306770245bf9aa37f3da253b352 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/cc8f4306770245bf9aa37f3da253b352 2024-04-14T11:17:50,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/cc8f4306770245bf9aa37f3da253b352, entries=150, sequenceid=305, filesize=12.0 K 2024-04-14T11:17:50,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/4e515c961afb41fd97c9fb9e083606eb as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4e515c961afb41fd97c9fb9e083606eb 2024-04-14T11:17:50,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4e515c961afb41fd97c9fb9e083606eb, entries=150, sequenceid=305, filesize=12.0 K 2024-04-14T11:17:50,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 498ms, sequenceid=305, compaction requested=true 2024-04-14T11:17:50,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:50,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:50,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:50,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:50,312 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:50,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:50,312 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:50,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:50,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:50,313 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103022 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:50,313 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/A is initiating minor compaction (all files) 2024-04-14T11:17:50,313 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/A in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:50,313 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37466 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:50,313 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/B is initiating minor compaction (all files) 2024-04-14T11:17:50,313 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/83bec1e9658d4c5cbd2de8d4292a6142, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/26a441709e84433e84e3cf08d7ede4a5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/35ef2a04a07b49129b92a2903c6a0b08] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=100.6 K 2024-04-14T11:17:50,313 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/B in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:50,313 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:50,313 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/124ae69d25f94e0bb566c62c9c9cbe45, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/60bdb660926b4605b0ad195129f852a8, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/cc8f4306770245bf9aa37f3da253b352] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=36.6 K 2024-04-14T11:17:50,313 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/83bec1e9658d4c5cbd2de8d4292a6142, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/26a441709e84433e84e3cf08d7ede4a5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/35ef2a04a07b49129b92a2903c6a0b08] 2024-04-14T11:17:50,314 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 83bec1e9658d4c5cbd2de8d4292a6142, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1713093468452 2024-04-14T11:17:50,314 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 124ae69d25f94e0bb566c62c9c9cbe45, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1713093468452 2024-04-14T11:17:50,315 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 26a441709e84433e84e3cf08d7ede4a5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1713093468562 2024-04-14T11:17:50,315 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 60bdb660926b4605b0ad195129f852a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1713093468562 2024-04-14T11:17:50,315 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 35ef2a04a07b49129b92a2903c6a0b08, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1713093469194 2024-04-14T11:17:50,315 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting cc8f4306770245bf9aa37f3da253b352, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1713093469194 2024-04-14T11:17:50,324 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:50,325 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#B#compaction#610 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:50,326 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/eb438822d51049acacb2d606efb09d84 is 50, key is test_row_0/B:col10/1713093469200/Put/seqid=0 2024-04-14T11:17:50,329 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20240414efb996d274274865bcce573b17a7f8ff_d2b4a72b9f8ab01eabcbb9f313e8e868 store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:50,331 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20240414efb996d274274865bcce573b17a7f8ff_d2b4a72b9f8ab01eabcbb9f313e8e868, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:50,331 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414efb996d274274865bcce573b17a7f8ff_d2b4a72b9f8ab01eabcbb9f313e8e868 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:50,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=182 2024-04-14T11:17:50,336 INFO [Thread-2769 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 182 completed 2024-04-14T11:17:50,338 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:50,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=184, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees 2024-04-14T11:17:50,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=184 2024-04-14T11:17:50,343 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=184, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:50,344 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=184, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:50,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=185, ppid=184, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:50,349 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/eb438822d51049acacb2d606efb09d84 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/eb438822d51049acacb2d606efb09d84 2024-04-14T11:17:50,352 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#A#compaction#609 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:50,352 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/4648a11f0ce9431baddfbc81b060ff24 is 175, key is test_row_0/A:col10/1713093469200/Put/seqid=0 2024-04-14T11:17:50,357 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/B of d2b4a72b9f8ab01eabcbb9f313e8e868 into eb438822d51049acacb2d606efb09d84(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:50,357 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:50,357 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/B, priority=13, startTime=1713093470312; duration=0sec 2024-04-14T11:17:50,357 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:50,357 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:B 2024-04-14T11:17:50,357 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:50,359 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37432 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:50,359 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/C is initiating minor compaction (all files) 2024-04-14T11:17:50,359 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/C in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:50,359 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/7e6e9fbf44c548afaf73ca2d3988b40f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/22a91df9f9cc4b6388de2f7c0b1cca01, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4e515c961afb41fd97c9fb9e083606eb] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=36.6 K 2024-04-14T11:17:50,360 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 7e6e9fbf44c548afaf73ca2d3988b40f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1713093468452 2024-04-14T11:17:50,360 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 22a91df9f9cc4b6388de2f7c0b1cca01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1713093468562 2024-04-14T11:17:50,361 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 4e515c961afb41fd97c9fb9e083606eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1713093469194 2024-04-14T11:17:50,370 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#C#compaction#611 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:50,371 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/5b9cac0f91254ba6aa745adada4b4d68 is 50, key is test_row_0/C:col10/1713093469200/Put/seqid=0 2024-04-14T11:17:50,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=184 2024-04-14T11:17:50,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:50,441 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:50,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-04-14T11:17:50,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414edc35621a3934cb9b98b93a3378b9427_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093470441/Put/seqid=0 2024-04-14T11:17:50,464 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414edc35621a3934cb9b98b93a3378b9427_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414edc35621a3934cb9b98b93a3378b9427_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:50,465 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/67c84de37e164a50978d37fc6ca4f66f, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:50,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/67c84de37e164a50978d37fc6ca4f66f is 175, key is test_row_0/A:col10/1713093470441/Put/seqid=0 2024-04-14T11:17:50,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093530469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093530471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093530472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093530473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,476 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/67c84de37e164a50978d37fc6ca4f66f 2024-04-14T11:17:50,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093530473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/3560bd0048ff492bb923291e8dd6cfaf is 50, key is test_row_0/B:col10/1713093470441/Put/seqid=0 2024-04-14T11:17:50,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/3560bd0048ff492bb923291e8dd6cfaf 2024-04-14T11:17:50,497 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-04-14T11:17:50,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:50,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:50,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:50,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(53): pid=185 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:50,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:50,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/1244fb2e29af422cac702c603d341c31 is 50, key is test_row_0/C:col10/1713093470441/Put/seqid=0 2024-04-14T11:17:50,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/1244fb2e29af422cac702c603d341c31 2024-04-14T11:17:50,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/67c84de37e164a50978d37fc6ca4f66f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/67c84de37e164a50978d37fc6ca4f66f 2024-04-14T11:17:50,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/67c84de37e164a50978d37fc6ca4f66f, entries=150, sequenceid=324, filesize=30.5 K 2024-04-14T11:17:50,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/3560bd0048ff492bb923291e8dd6cfaf as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/3560bd0048ff492bb923291e8dd6cfaf 2024-04-14T11:17:50,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/3560bd0048ff492bb923291e8dd6cfaf, entries=150, sequenceid=324, filesize=12.0 K 2024-04-14T11:17:50,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/1244fb2e29af422cac702c603d341c31 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/1244fb2e29af422cac702c603d341c31 2024-04-14T11:17:50,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/1244fb2e29af422cac702c603d341c31, entries=150, sequenceid=324, filesize=12.0 K 2024-04-14T11:17:50,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=120.76 KB/123660 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 97ms, sequenceid=324, compaction requested=false 2024-04-14T11:17:50,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:50,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:50,576 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:50,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-04-14T11:17:50,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414c4cda27c090d4f53a890d5ef92e17f10_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093470471/Put/seqid=0 2024-04-14T11:17:50,594 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414c4cda27c090d4f53a890d5ef92e17f10_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414c4cda27c090d4f53a890d5ef92e17f10_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:50,596 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/c03b42392817475a8e135fe8915396ee, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:50,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093530592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/c03b42392817475a8e135fe8915396ee is 175, key is test_row_0/A:col10/1713093470471/Put/seqid=0 2024-04-14T11:17:50,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093530593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093530593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093530596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093530597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=184 2024-04-14T11:17:50,649 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-04-14T11:17:50,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:50,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:50,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:50,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(53): pid=185 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:50,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:50,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093530698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093530698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093530698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093530701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093530702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,764 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/4648a11f0ce9431baddfbc81b060ff24 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/4648a11f0ce9431baddfbc81b060ff24 2024-04-14T11:17:50,769 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/A of d2b4a72b9f8ab01eabcbb9f313e8e868 into 4648a11f0ce9431baddfbc81b060ff24(size=31.2 K), total size for store is 61.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:50,770 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:50,770 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/A, priority=13, startTime=1713093470312; duration=0sec 2024-04-14T11:17:50,770 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:50,770 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:A 2024-04-14T11:17:50,785 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/5b9cac0f91254ba6aa745adada4b4d68 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/5b9cac0f91254ba6aa745adada4b4d68 2024-04-14T11:17:50,791 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/C of d2b4a72b9f8ab01eabcbb9f313e8e868 into 5b9cac0f91254ba6aa745adada4b4d68(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:50,792 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:50,792 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/C, priority=13, startTime=1713093470312; duration=0sec 2024-04-14T11:17:50,792 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:50,792 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:C 2024-04-14T11:17:50,802 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-04-14T11:17:50,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:50,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:50,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:50,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(53): pid=185 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:50,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:50,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093530902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093530903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093530903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093530906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093530907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=184 2024-04-14T11:17:50,955 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:50,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-04-14T11:17:50,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:50,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:50,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:50,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(53): pid=185 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:50,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:51,003 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=346, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/c03b42392817475a8e135fe8915396ee 2024-04-14T11:17:51,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/7afb7c149ef54305b0b32d5b74791b29 is 50, key is test_row_0/B:col10/1713093470471/Put/seqid=0 2024-04-14T11:17:51,108 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:51,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-04-14T11:17:51,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:51,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:51,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:51,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(53): pid=185 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:51,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:51,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093531205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:51,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093531206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:51,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093531207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:51,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093531210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:51,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093531212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:51,261 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:51,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-04-14T11:17:51,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:51,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:51,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:51,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(53): pid=185 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:51,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:51,414 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:51,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-04-14T11:17:51,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:51,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:51,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:51,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] handler.RSProcedureHandler(53): pid=185 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:51,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=185 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=185 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:51,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/7afb7c149ef54305b0b32d5b74791b29 2024-04-14T11:17:51,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/fc1e2b50dc07453ab6eb24a628339f28 is 50, key is test_row_0/C:col10/1713093470471/Put/seqid=0 2024-04-14T11:17:51,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/fc1e2b50dc07453ab6eb24a628339f28 2024-04-14T11:17:51,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/c03b42392817475a8e135fe8915396ee as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c03b42392817475a8e135fe8915396ee 2024-04-14T11:17:51,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c03b42392817475a8e135fe8915396ee, entries=150, sequenceid=346, filesize=30.5 K 2024-04-14T11:17:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=184 2024-04-14T11:17:51,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/7afb7c149ef54305b0b32d5b74791b29 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/7afb7c149ef54305b0b32d5b74791b29 2024-04-14T11:17:51,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/7afb7c149ef54305b0b32d5b74791b29, entries=150, sequenceid=346, filesize=12.0 K 2024-04-14T11:17:51,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/fc1e2b50dc07453ab6eb24a628339f28 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/fc1e2b50dc07453ab6eb24a628339f28 2024-04-14T11:17:51,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/fc1e2b50dc07453ab6eb24a628339f28, entries=150, sequenceid=346, filesize=12.0 K 2024-04-14T11:17:51,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 882ms, sequenceid=346, compaction requested=true 2024-04-14T11:17:51,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:51,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:51,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:51,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:51,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:51,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:51,458 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:51,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:51,458 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:51,459 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37568 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:51,459 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/B is initiating minor compaction (all files) 2024-04-14T11:17:51,459 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/B in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:51,459 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/eb438822d51049acacb2d606efb09d84, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/3560bd0048ff492bb923291e8dd6cfaf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/7afb7c149ef54305b0b32d5b74791b29] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=36.7 K 2024-04-14T11:17:51,460 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94430 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:51,460 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/A is initiating minor compaction (all files) 2024-04-14T11:17:51,460 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/A in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:51,460 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/4648a11f0ce9431baddfbc81b060ff24, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/67c84de37e164a50978d37fc6ca4f66f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c03b42392817475a8e135fe8915396ee] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=92.2 K 2024-04-14T11:17:51,460 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:51,460 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/4648a11f0ce9431baddfbc81b060ff24, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/67c84de37e164a50978d37fc6ca4f66f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c03b42392817475a8e135fe8915396ee] 2024-04-14T11:17:51,460 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting eb438822d51049acacb2d606efb09d84, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1713093469194 2024-04-14T11:17:51,460 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4648a11f0ce9431baddfbc81b060ff24, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1713093469194 2024-04-14T11:17:51,461 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 3560bd0048ff492bb923291e8dd6cfaf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1713093470440 2024-04-14T11:17:51,461 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 67c84de37e164a50978d37fc6ca4f66f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1713093470440 2024-04-14T11:17:51,461 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 7afb7c149ef54305b0b32d5b74791b29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1713093470470 2024-04-14T11:17:51,461 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c03b42392817475a8e135fe8915396ee, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1713093470470 2024-04-14T11:17:51,469 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:51,470 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#B#compaction#619 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:51,470 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/c4c7df5b0cee4bfca73df705323a85c5 is 50, key is test_row_0/B:col10/1713093470471/Put/seqid=0 2024-04-14T11:17:51,472 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20240414ab3d611da3564f4c95d63ef8c55a5eff_d2b4a72b9f8ab01eabcbb9f313e8e868 store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:51,474 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20240414ab3d611da3564f4c95d63ef8c55a5eff_d2b4a72b9f8ab01eabcbb9f313e8e868, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:51,474 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414ab3d611da3564f4c95d63ef8c55a5eff_d2b4a72b9f8ab01eabcbb9f313e8e868 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:51,484 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#A#compaction#618 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:51,485 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/453e3fdbf17f4d87baec86040f3bd464 is 175, key is test_row_0/A:col10/1713093470471/Put/seqid=0 2024-04-14T11:17:51,488 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/c4c7df5b0cee4bfca73df705323a85c5 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/c4c7df5b0cee4bfca73df705323a85c5 2024-04-14T11:17:51,496 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/B of d2b4a72b9f8ab01eabcbb9f313e8e868 into c4c7df5b0cee4bfca73df705323a85c5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:51,496 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:51,496 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/B, priority=13, startTime=1713093471458; duration=0sec 2024-04-14T11:17:51,496 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:51,496 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:B 2024-04-14T11:17:51,496 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:51,497 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/453e3fdbf17f4d87baec86040f3bd464 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/453e3fdbf17f4d87baec86040f3bd464 2024-04-14T11:17:51,498 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37534 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:51,498 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/C is initiating minor compaction (all files) 2024-04-14T11:17:51,498 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/C in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:51,498 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/5b9cac0f91254ba6aa745adada4b4d68, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/1244fb2e29af422cac702c603d341c31, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/fc1e2b50dc07453ab6eb24a628339f28] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=36.7 K 2024-04-14T11:17:51,498 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 5b9cac0f91254ba6aa745adada4b4d68, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1713093469194 2024-04-14T11:17:51,499 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 1244fb2e29af422cac702c603d341c31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1713093470440 2024-04-14T11:17:51,499 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting fc1e2b50dc07453ab6eb24a628339f28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1713093470470 2024-04-14T11:17:51,504 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/A of d2b4a72b9f8ab01eabcbb9f313e8e868 into 453e3fdbf17f4d87baec86040f3bd464(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:51,504 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:51,504 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/A, priority=13, startTime=1713093471458; duration=0sec 2024-04-14T11:17:51,504 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:51,504 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:A 2024-04-14T11:17:51,507 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#C#compaction#620 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:51,508 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/279f5ce0f13a4987b896d9e93f84717c is 50, key is test_row_0/C:col10/1713093470471/Put/seqid=0 2024-04-14T11:17:51,519 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/279f5ce0f13a4987b896d9e93f84717c as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/279f5ce0f13a4987b896d9e93f84717c 2024-04-14T11:17:51,525 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/C of d2b4a72b9f8ab01eabcbb9f313e8e868 into 279f5ce0f13a4987b896d9e93f84717c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:51,525 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:51,525 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/C, priority=13, startTime=1713093471458; duration=0sec 2024-04-14T11:17:51,525 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:51,525 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:C 2024-04-14T11:17:51,582 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:51,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=185 2024-04-14T11:17:51,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:51,583 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-04-14T11:17:51,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404149f6d2edae6bf4293a86e6a2339316396_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093470592/Put/seqid=0 2024-04-14T11:17:51,599 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404149f6d2edae6bf4293a86e6a2339316396_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404149f6d2edae6bf4293a86e6a2339316396_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:51,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/7c6f8c6afb1d488fab4e4e664b5f0145, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:51,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/7c6f8c6afb1d488fab4e4e664b5f0145 is 175, key is test_row_0/A:col10/1713093470592/Put/seqid=0 2024-04-14T11:17:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:51,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:51,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093531726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:51,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093531727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:51,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:51,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093531727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093531728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:51,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093531729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:51,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093531832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:51,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:51,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093531832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093531832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:51,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093531832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:51,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093531832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,014 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/7c6f8c6afb1d488fab4e4e664b5f0145 2024-04-14T11:17:52,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/03d03e7895204845856ba4aed70141ad is 50, key is test_row_0/B:col10/1713093470592/Put/seqid=0 2024-04-14T11:17:52,026 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/03d03e7895204845856ba4aed70141ad 2024-04-14T11:17:52,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/c3101beb67874ca2b67fb4a6902ced54 is 50, key is test_row_0/C:col10/1713093470592/Put/seqid=0 2024-04-14T11:17:52,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093532035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093532035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093532035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093532036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093532036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,040 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/c3101beb67874ca2b67fb4a6902ced54 2024-04-14T11:17:52,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/7c6f8c6afb1d488fab4e4e664b5f0145 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/7c6f8c6afb1d488fab4e4e664b5f0145 2024-04-14T11:17:52,051 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/7c6f8c6afb1d488fab4e4e664b5f0145, entries=150, sequenceid=365, filesize=30.5 K 2024-04-14T11:17:52,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/03d03e7895204845856ba4aed70141ad as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/03d03e7895204845856ba4aed70141ad 2024-04-14T11:17:52,057 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/03d03e7895204845856ba4aed70141ad, entries=150, sequenceid=365, filesize=12.0 K 2024-04-14T11:17:52,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/c3101beb67874ca2b67fb4a6902ced54 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c3101beb67874ca2b67fb4a6902ced54 2024-04-14T11:17:52,063 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c3101beb67874ca2b67fb4a6902ced54, entries=150, sequenceid=365, filesize=12.0 K 2024-04-14T11:17:52,064 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(3035): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 481ms, sequenceid=365, compaction requested=false 2024-04-14T11:17:52,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:52,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:52,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=185}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=185 2024-04-14T11:17:52,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=185 2024-04-14T11:17:52,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=185, resume processing ppid=184 2024-04-14T11:17:52,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=185, ppid=184, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7210 sec 2024-04-14T11:17:52,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=184, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=184, table=TestAcidGuarantees in 1.7290 sec 2024-04-14T11:17:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:52,340 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:52,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-04-14T11:17:52,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414396bfc3dca63481c804206c3969fd9c2_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093471726/Put/seqid=0 2024-04-14T11:17:52,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093532347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093532347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093532348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093532349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093532349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,356 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414396bfc3dca63481c804206c3969fd9c2_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414396bfc3dca63481c804206c3969fd9c2_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:52,357 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/36260ca37b9b46febe95528ce4512b59, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:52,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/36260ca37b9b46febe95528ce4512b59 is 175, key is test_row_0/A:col10/1713093471726/Put/seqid=0 2024-04-14T11:17:52,363 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=389, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/36260ca37b9b46febe95528ce4512b59 2024-04-14T11:17:52,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/63cafd1594dc4ce891e2b47f75b8e98d is 50, key is test_row_0/B:col10/1713093471726/Put/seqid=0 2024-04-14T11:17:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=184 2024-04-14T11:17:52,447 INFO [Thread-2769 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 184 completed 2024-04-14T11:17:52,449 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=186, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=186, table=TestAcidGuarantees 2024-04-14T11:17:52,452 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=186, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=186, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=186 2024-04-14T11:17:52,453 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(88): pid=186, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=186, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:52,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=187, ppid=186, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:52,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093532453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093532453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093532453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093532455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093532455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=186 2024-04-14T11:17:52,605 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2024-04-14T11:17:52,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:52,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:52,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:52,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:52,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:52,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093532655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093532655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093532655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093532658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093532658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=186 2024-04-14T11:17:52,759 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2024-04-14T11:17:52,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:52,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:52,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:52,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] handler.RSProcedureHandler(53): pid=187 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:52,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=187 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:52,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=187 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:52,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/63cafd1594dc4ce891e2b47f75b8e98d 2024-04-14T11:17:52,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/b5ec5c508a65483e82600441be29e114 is 50, key is test_row_0/C:col10/1713093471726/Put/seqid=0 2024-04-14T11:17:52,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/b5ec5c508a65483e82600441be29e114 2024-04-14T11:17:52,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/36260ca37b9b46febe95528ce4512b59 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/36260ca37b9b46febe95528ce4512b59 2024-04-14T11:17:52,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/36260ca37b9b46febe95528ce4512b59, entries=200, sequenceid=389, filesize=39.0 K 2024-04-14T11:17:52,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/63cafd1594dc4ce891e2b47f75b8e98d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/63cafd1594dc4ce891e2b47f75b8e98d 2024-04-14T11:17:52,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/63cafd1594dc4ce891e2b47f75b8e98d, entries=150, sequenceid=389, filesize=12.0 K 2024-04-14T11:17:52,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/b5ec5c508a65483e82600441be29e114 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/b5ec5c508a65483e82600441be29e114 2024-04-14T11:17:52,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/b5ec5c508a65483e82600441be29e114, entries=150, sequenceid=389, filesize=12.0 K 2024-04-14T11:17:52,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 471ms, sequenceid=389, compaction requested=true 2024-04-14T11:17:52,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:52,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:52,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:52,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:52,811 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:52,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:52,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:52,811 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:52,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:52,813 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37670 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:52,813 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103226 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:52,813 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/B is initiating minor compaction (all files) 2024-04-14T11:17:52,813 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/A is initiating minor compaction (all files) 2024-04-14T11:17:52,813 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/A in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:52,813 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/B in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:52,813 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/c4c7df5b0cee4bfca73df705323a85c5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/03d03e7895204845856ba4aed70141ad, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/63cafd1594dc4ce891e2b47f75b8e98d] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=36.8 K 2024-04-14T11:17:52,813 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/453e3fdbf17f4d87baec86040f3bd464, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/7c6f8c6afb1d488fab4e4e664b5f0145, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/36260ca37b9b46febe95528ce4512b59] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=100.8 K 2024-04-14T11:17:52,813 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:52,813 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/453e3fdbf17f4d87baec86040f3bd464, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/7c6f8c6afb1d488fab4e4e664b5f0145, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/36260ca37b9b46febe95528ce4512b59] 2024-04-14T11:17:52,813 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting c4c7df5b0cee4bfca73df705323a85c5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1713093470470 2024-04-14T11:17:52,814 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 453e3fdbf17f4d87baec86040f3bd464, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1713093470470 2024-04-14T11:17:52,814 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 03d03e7895204845856ba4aed70141ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1713093470586 2024-04-14T11:17:52,814 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 7c6f8c6afb1d488fab4e4e664b5f0145, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1713093470586 2024-04-14T11:17:52,815 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 63cafd1594dc4ce891e2b47f75b8e98d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1713093471726 2024-04-14T11:17:52,815 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 36260ca37b9b46febe95528ce4512b59, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1713093471725 2024-04-14T11:17:52,822 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:52,823 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#B#compaction#627 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:52,824 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/8c2436508f6c40b0a5d34d590f5604d6 is 50, key is test_row_0/B:col10/1713093471726/Put/seqid=0 2024-04-14T11:17:52,824 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20240414e1771df5d59f41f7a0d9279fd76a627d_d2b4a72b9f8ab01eabcbb9f313e8e868 store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:52,826 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20240414e1771df5d59f41f7a0d9279fd76a627d_d2b4a72b9f8ab01eabcbb9f313e8e868, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:52,826 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414e1771df5d59f41f7a0d9279fd76a627d_d2b4a72b9f8ab01eabcbb9f313e8e868 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:52,848 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#A#compaction#628 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:52,849 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/a4857fc536dc4deb83d7ec328e09cf02 is 175, key is test_row_0/A:col10/1713093471726/Put/seqid=0 2024-04-14T11:17:52,912 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=187 2024-04-14T11:17:52,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:52,913 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-04-14T11:17:52,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414a9059ea6753f4affae20c363f7467f75_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093472348/Put/seqid=0 2024-04-14T11:17:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:52,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:52,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093532975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093532974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093532975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093532979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093532980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:53,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=186 2024-04-14T11:17:53,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:53,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093533081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:53,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:53,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093533081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:53,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:53,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093533081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:53,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:53,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093533084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:53,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:53,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093533085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:53,238 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/8c2436508f6c40b0a5d34d590f5604d6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/8c2436508f6c40b0a5d34d590f5604d6 2024-04-14T11:17:53,244 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/B of d2b4a72b9f8ab01eabcbb9f313e8e868 into 8c2436508f6c40b0a5d34d590f5604d6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:53,244 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:53,244 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/B, priority=13, startTime=1713093472811; duration=0sec 2024-04-14T11:17:53,244 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:53,244 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:B 2024-04-14T11:17:53,244 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:53,250 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37636 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:53,250 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/C is initiating minor compaction (all files) 2024-04-14T11:17:53,250 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/C in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:53,250 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/279f5ce0f13a4987b896d9e93f84717c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c3101beb67874ca2b67fb4a6902ced54, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/b5ec5c508a65483e82600441be29e114] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=36.8 K 2024-04-14T11:17:53,250 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 279f5ce0f13a4987b896d9e93f84717c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1713093470470 2024-04-14T11:17:53,251 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting c3101beb67874ca2b67fb4a6902ced54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1713093470586 2024-04-14T11:17:53,251 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting b5ec5c508a65483e82600441be29e114, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1713093471726 2024-04-14T11:17:53,260 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#C#compaction#630 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:53,261 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/87ba252b3c334c78b6096eb8a07cb0d0 is 50, key is test_row_0/C:col10/1713093471726/Put/seqid=0 2024-04-14T11:17:53,263 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/a4857fc536dc4deb83d7ec328e09cf02 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/a4857fc536dc4deb83d7ec328e09cf02 2024-04-14T11:17:53,269 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/A of d2b4a72b9f8ab01eabcbb9f313e8e868 into a4857fc536dc4deb83d7ec328e09cf02(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:53,269 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:53,269 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/A, priority=13, startTime=1713093472811; duration=0sec 2024-04-14T11:17:53,269 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:53,269 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:A 2024-04-14T11:17:53,272 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/87ba252b3c334c78b6096eb8a07cb0d0 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/87ba252b3c334c78b6096eb8a07cb0d0 2024-04-14T11:17:53,278 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/C of d2b4a72b9f8ab01eabcbb9f313e8e868 into 87ba252b3c334c78b6096eb8a07cb0d0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:53,278 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:53,278 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/C, priority=13, startTime=1713093472811; duration=0sec 2024-04-14T11:17:53,278 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:53,278 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:C 2024-04-14T11:17:53,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:53,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093533283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:53,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:53,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093533283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:53,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:53,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093533283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:53,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:53,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093533287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:53,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:53,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093533287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:53,329 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414a9059ea6753f4affae20c363f7467f75_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414a9059ea6753f4affae20c363f7467f75_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:53,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/0fca0158146a41628f60c9778a76dd5c, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:53,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/0fca0158146a41628f60c9778a76dd5c is 175, key is test_row_0/A:col10/1713093472348/Put/seqid=0 2024-04-14T11:17:53,335 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=402, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/0fca0158146a41628f60c9778a76dd5c 2024-04-14T11:17:53,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/9b0eaee2d0804767a83f108b7f5adf8f is 50, key is test_row_0/B:col10/1713093472348/Put/seqid=0 2024-04-14T11:17:53,347 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/9b0eaee2d0804767a83f108b7f5adf8f 2024-04-14T11:17:53,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/785d7230b3be4c0c8536dffd3e9082fa is 50, key is test_row_0/C:col10/1713093472348/Put/seqid=0 2024-04-14T11:17:53,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=186 2024-04-14T11:17:53,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:53,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093533585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:53,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093533587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:53,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:53,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093533588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:53,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093533590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:53,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:53,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093533590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:53,760 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/785d7230b3be4c0c8536dffd3e9082fa 2024-04-14T11:17:53,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/0fca0158146a41628f60c9778a76dd5c as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/0fca0158146a41628f60c9778a76dd5c 2024-04-14T11:17:53,771 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/0fca0158146a41628f60c9778a76dd5c, entries=150, sequenceid=402, filesize=30.5 K 2024-04-14T11:17:53,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/9b0eaee2d0804767a83f108b7f5adf8f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/9b0eaee2d0804767a83f108b7f5adf8f 2024-04-14T11:17:53,776 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/9b0eaee2d0804767a83f108b7f5adf8f, entries=150, sequenceid=402, filesize=12.0 K 2024-04-14T11:17:53,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/785d7230b3be4c0c8536dffd3e9082fa as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/785d7230b3be4c0c8536dffd3e9082fa 2024-04-14T11:17:53,782 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/785d7230b3be4c0c8536dffd3e9082fa, entries=150, sequenceid=402, filesize=12.0 K 2024-04-14T11:17:53,782 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(3035): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 869ms, sequenceid=402, compaction requested=false 2024-04-14T11:17:53,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:53,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:53,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=187}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=187 2024-04-14T11:17:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=187 2024-04-14T11:17:53,785 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=187, resume processing ppid=186 2024-04-14T11:17:53,785 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1462): Finished pid=187, ppid=186, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3310 sec 2024-04-14T11:17:53,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=186, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=186, table=TestAcidGuarantees in 1.3370 sec 2024-04-14T11:17:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:54,091 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:54,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-04-14T11:17:54,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414e939659a8a47432d856632a4585a2a7b_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093474089/Put/seqid=0 2024-04-14T11:17:54,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093534099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093534099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093534100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093534100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093534101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,105 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414e939659a8a47432d856632a4585a2a7b_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414e939659a8a47432d856632a4585a2a7b_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:54,107 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/389b57bac8ef49cdb2814d2f7f7ed3a3, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:54,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/389b57bac8ef49cdb2814d2f7f7ed3a3 is 175, key is test_row_0/A:col10/1713093474089/Put/seqid=0 2024-04-14T11:17:54,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093534203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093534204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093534204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093534205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093534205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093534407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093534407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093534407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093534408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093534408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,512 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=429, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/389b57bac8ef49cdb2814d2f7f7ed3a3 2024-04-14T11:17:54,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/04440446f8a34484bcb381b47c9018a6 is 50, key is test_row_0/B:col10/1713093474089/Put/seqid=0 2024-04-14T11:17:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=186 2024-04-14T11:17:54,558 INFO [Thread-2769 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 186 completed 2024-04-14T11:17:54,560 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=188, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=188, table=TestAcidGuarantees 2024-04-14T11:17:54,563 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=188, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=188, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=188 2024-04-14T11:17:54,564 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(88): pid=188, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=188, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:54,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=189, ppid=188, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=188 2024-04-14T11:17:54,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093534711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093534711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093534713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093534713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,716 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=189 2024-04-14T11:17:54,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:54,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:54,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:54,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] handler.RSProcedureHandler(53): pid=189 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:54,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=189 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=189 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:54,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093534723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=188 2024-04-14T11:17:54,873 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:54,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=189 2024-04-14T11:17:54,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:54,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:54,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:54,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] handler.RSProcedureHandler(53): pid=189 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:54,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=189 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=189 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:54,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/04440446f8a34484bcb381b47c9018a6 2024-04-14T11:17:54,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/3626796d5d6e464eb080fd2184a1c4f7 is 50, key is test_row_0/C:col10/1713093474089/Put/seqid=0 2024-04-14T11:17:54,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/3626796d5d6e464eb080fd2184a1c4f7 2024-04-14T11:17:54,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/389b57bac8ef49cdb2814d2f7f7ed3a3 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/389b57bac8ef49cdb2814d2f7f7ed3a3 2024-04-14T11:17:54,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/389b57bac8ef49cdb2814d2f7f7ed3a3, entries=200, sequenceid=429, filesize=39.0 K 2024-04-14T11:17:54,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/04440446f8a34484bcb381b47c9018a6 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/04440446f8a34484bcb381b47c9018a6 2024-04-14T11:17:54,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/04440446f8a34484bcb381b47c9018a6, entries=150, sequenceid=429, filesize=12.0 K 2024-04-14T11:17:54,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/3626796d5d6e464eb080fd2184a1c4f7 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/3626796d5d6e464eb080fd2184a1c4f7 2024-04-14T11:17:54,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/3626796d5d6e464eb080fd2184a1c4f7, entries=150, sequenceid=429, filesize=12.0 K 2024-04-14T11:17:54,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 891ms, sequenceid=429, compaction requested=true 2024-04-14T11:17:54,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:54,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:54,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:54,983 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:54,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:54,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:54,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:54,983 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:54,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:54,984 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103328 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:54,984 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/A is initiating minor compaction (all files) 2024-04-14T11:17:54,984 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/A in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:54,984 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37772 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:54,984 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/a4857fc536dc4deb83d7ec328e09cf02, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/0fca0158146a41628f60c9778a76dd5c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/389b57bac8ef49cdb2814d2f7f7ed3a3] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=100.9 K 2024-04-14T11:17:54,984 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:54,984 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/B is initiating minor compaction (all files) 2024-04-14T11:17:54,984 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/a4857fc536dc4deb83d7ec328e09cf02, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/0fca0158146a41628f60c9778a76dd5c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/389b57bac8ef49cdb2814d2f7f7ed3a3] 2024-04-14T11:17:54,984 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/B in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:54,984 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/8c2436508f6c40b0a5d34d590f5604d6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/9b0eaee2d0804767a83f108b7f5adf8f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/04440446f8a34484bcb381b47c9018a6] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=36.9 K 2024-04-14T11:17:54,985 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting a4857fc536dc4deb83d7ec328e09cf02, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1713093471726 2024-04-14T11:17:54,985 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 8c2436508f6c40b0a5d34d590f5604d6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1713093471726 2024-04-14T11:17:54,985 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 0fca0158146a41628f60c9778a76dd5c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1713093472343 2024-04-14T11:17:54,985 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 9b0eaee2d0804767a83f108b7f5adf8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1713093472343 2024-04-14T11:17:54,986 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 389b57bac8ef49cdb2814d2f7f7ed3a3, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1713093472969 2024-04-14T11:17:54,986 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 04440446f8a34484bcb381b47c9018a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1713093472978 2024-04-14T11:17:54,995 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:54,996 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#B#compaction#636 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:54,996 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/d344a462af5a442695a2db2d03c7f4fd is 50, key is test_row_0/B:col10/1713093474089/Put/seqid=0 2024-04-14T11:17:54,996 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202404141fb13edf4f564aa09fb4ee49b7580a10_d2b4a72b9f8ab01eabcbb9f313e8e868 store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:54,998 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202404141fb13edf4f564aa09fb4ee49b7580a10_d2b4a72b9f8ab01eabcbb9f313e8e868, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:54,998 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404141fb13edf4f564aa09fb4ee49b7580a10_d2b4a72b9f8ab01eabcbb9f313e8e868 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:55,016 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/d344a462af5a442695a2db2d03c7f4fd as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/d344a462af5a442695a2db2d03c7f4fd 2024-04-14T11:17:55,020 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#A#compaction#637 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:55,020 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/f747662db1a44af4949fb422ff2b9dc3 is 175, key is test_row_0/A:col10/1713093474089/Put/seqid=0 2024-04-14T11:17:55,024 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/B of d2b4a72b9f8ab01eabcbb9f313e8e868 into d344a462af5a442695a2db2d03c7f4fd(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:55,024 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:55,024 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/B, priority=13, startTime=1713093474983; duration=0sec 2024-04-14T11:17:55,024 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:55,024 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:B 2024-04-14T11:17:55,024 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:55,026 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,027 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37738 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:55,027 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/C is initiating minor compaction (all files) 2024-04-14T11:17:55,027 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/C in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:55,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=189 2024-04-14T11:17:55,027 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/87ba252b3c334c78b6096eb8a07cb0d0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/785d7230b3be4c0c8536dffd3e9082fa, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/3626796d5d6e464eb080fd2184a1c4f7] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=36.9 K 2024-04-14T11:17:55,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:55,027 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-04-14T11:17:55,027 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 87ba252b3c334c78b6096eb8a07cb0d0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1713093471726 2024-04-14T11:17:55,028 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 785d7230b3be4c0c8536dffd3e9082fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1713093472343 2024-04-14T11:17:55,029 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 3626796d5d6e464eb080fd2184a1c4f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1713093472978 2024-04-14T11:17:55,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404141e081cccb3204057b4770c78a3c697a6_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093474099/Put/seqid=0 2024-04-14T11:17:55,038 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#C#compaction#639 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:55,038 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/c2a96eeb8c5543c1aad7361f653dc87e is 50, key is test_row_0/C:col10/1713093474089/Put/seqid=0 2024-04-14T11:17:55,054 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404141e081cccb3204057b4770c78a3c697a6_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404141e081cccb3204057b4770c78a3c697a6_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:55,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/1fa353e3563848089ae15d60839b2107, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:55,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/1fa353e3563848089ae15d60839b2107 is 175, key is test_row_0/A:col10/1713093474099/Put/seqid=0 2024-04-14T11:17:55,057 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/c2a96eeb8c5543c1aad7361f653dc87e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c2a96eeb8c5543c1aad7361f653dc87e 2024-04-14T11:17:55,063 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=442, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/1fa353e3563848089ae15d60839b2107 2024-04-14T11:17:55,065 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/C of d2b4a72b9f8ab01eabcbb9f313e8e868 into c2a96eeb8c5543c1aad7361f653dc87e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:55,065 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:55,065 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/C, priority=13, startTime=1713093474983; duration=0sec 2024-04-14T11:17:55,065 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:55,065 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:C 2024-04-14T11:17:55,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/17458b397779414b8b578282b3905b10 is 50, key is test_row_0/B:col10/1713093474099/Put/seqid=0 2024-04-14T11:17:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=188 2024-04-14T11:17:55,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:55,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:55,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093535233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093535236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093535237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093535237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093535237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093535338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093535340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093535340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093535340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093535340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,432 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/f747662db1a44af4949fb422ff2b9dc3 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/f747662db1a44af4949fb422ff2b9dc3 2024-04-14T11:17:55,438 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/A of d2b4a72b9f8ab01eabcbb9f313e8e868 into f747662db1a44af4949fb422ff2b9dc3(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:55,438 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:55,438 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/A, priority=13, startTime=1713093474983; duration=0sec 2024-04-14T11:17:55,438 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:55,438 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:A 2024-04-14T11:17:55,477 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/17458b397779414b8b578282b3905b10 2024-04-14T11:17:55,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/4076bdec7c6045a49eed3d3e870dd4ea is 50, key is test_row_0/C:col10/1713093474099/Put/seqid=0 2024-04-14T11:17:55,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093535541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093535542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093535543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093535544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093535544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=188 2024-04-14T11:17:55,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093535844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093535844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093535845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093535846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093535861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:55,889 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/4076bdec7c6045a49eed3d3e870dd4ea 2024-04-14T11:17:55,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/1fa353e3563848089ae15d60839b2107 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1fa353e3563848089ae15d60839b2107 2024-04-14T11:17:55,903 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1fa353e3563848089ae15d60839b2107, entries=150, sequenceid=442, filesize=30.5 K 2024-04-14T11:17:55,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/17458b397779414b8b578282b3905b10 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/17458b397779414b8b578282b3905b10 2024-04-14T11:17:55,909 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/17458b397779414b8b578282b3905b10, entries=150, sequenceid=442, filesize=12.0 K 2024-04-14T11:17:55,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/4076bdec7c6045a49eed3d3e870dd4ea as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4076bdec7c6045a49eed3d3e870dd4ea 2024-04-14T11:17:55,915 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4076bdec7c6045a49eed3d3e870dd4ea, entries=150, sequenceid=442, filesize=12.0 K 2024-04-14T11:17:55,916 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(3035): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 889ms, sequenceid=442, compaction requested=false 2024-04-14T11:17:55,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:55,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:55,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=189}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=189 2024-04-14T11:17:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=189 2024-04-14T11:17:55,918 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=189, resume processing ppid=188 2024-04-14T11:17:55,918 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=189, ppid=188, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3530 sec 2024-04-14T11:17:55,920 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=188, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=188, table=TestAcidGuarantees in 1.3590 sec 2024-04-14T11:17:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:56,349 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:56,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-04-14T11:17:56,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414fc745d1180d0466dbe45d5a58c3d1276_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093476349/Put/seqid=0 2024-04-14T11:17:56,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:56,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093536354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:56,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093536366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:56,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093536366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:56,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:56,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093536374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093536374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:56,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093536470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:56,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:56,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093536474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:56,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093536478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:56,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:56,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093536478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=188 2024-04-14T11:17:56,669 INFO [Thread-2769 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 188 completed 2024-04-14T11:17:56,671 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=190, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=190, table=TestAcidGuarantees 2024-04-14T11:17:56,673 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=190, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=190, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=190 2024-04-14T11:17:56,674 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(88): pid=190, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=190, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:56,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=191, ppid=190, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:56,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093536674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:56,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093536677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:56,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093536681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:56,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093536681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:56,765 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414fc745d1180d0466dbe45d5a58c3d1276_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414fc745d1180d0466dbe45d5a58c3d1276_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:56,766 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/c73db47d0b1a49df84c95a25ef22b7c2, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:56,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/c73db47d0b1a49df84c95a25ef22b7c2 is 175, key is test_row_0/A:col10/1713093476349/Put/seqid=0 2024-04-14T11:17:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=190 2024-04-14T11:17:56,826 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:56,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=191 2024-04-14T11:17:56,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:56,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:56,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:56,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] handler.RSProcedureHandler(53): pid=191 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:56,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=191 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=191 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=190 2024-04-14T11:17:56,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093536976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:56,978 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:56,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=191 2024-04-14T11:17:56,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:56,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:56,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:56,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] handler.RSProcedureHandler(53): pid=191 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:56,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=191 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=191 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:56,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093536980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:56,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093536984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:56,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093536984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:57,131 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:57,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=191 2024-04-14T11:17:57,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:57,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:57,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:57,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] handler.RSProcedureHandler(53): pid=191 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:57,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=191 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=191 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:57,173 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=473, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/c73db47d0b1a49df84c95a25ef22b7c2 2024-04-14T11:17:57,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/7342793dd55249089a7289f24efa2e7b is 50, key is test_row_0/B:col10/1713093476349/Put/seqid=0 2024-04-14T11:17:57,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/7342793dd55249089a7289f24efa2e7b 2024-04-14T11:17:57,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/c59fc433a92d43f7a1ad84b085a60d21 is 50, key is test_row_0/C:col10/1713093476349/Put/seqid=0 2024-04-14T11:17:57,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/c59fc433a92d43f7a1ad84b085a60d21 2024-04-14T11:17:57,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/c73db47d0b1a49df84c95a25ef22b7c2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c73db47d0b1a49df84c95a25ef22b7c2 2024-04-14T11:17:57,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c73db47d0b1a49df84c95a25ef22b7c2, entries=250, sequenceid=473, filesize=47.5 K 2024-04-14T11:17:57,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/7342793dd55249089a7289f24efa2e7b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/7342793dd55249089a7289f24efa2e7b 2024-04-14T11:17:57,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/7342793dd55249089a7289f24efa2e7b, entries=150, sequenceid=473, filesize=12.0 K 2024-04-14T11:17:57,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/c59fc433a92d43f7a1ad84b085a60d21 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c59fc433a92d43f7a1ad84b085a60d21 2024-04-14T11:17:57,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c59fc433a92d43f7a1ad84b085a60d21, entries=150, sequenceid=473, filesize=12.0 K 2024-04-14T11:17:57,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=40.25 KB/41220 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 871ms, sequenceid=473, compaction requested=true 2024-04-14T11:17:57,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:57,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:57,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:57,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:57,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:57,220 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:57,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:57,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:57,220 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:57,221 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37874 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:57,221 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/B is initiating minor compaction (all files) 2024-04-14T11:17:57,221 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/B in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:57,221 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/d344a462af5a442695a2db2d03c7f4fd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/17458b397779414b8b578282b3905b10, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/7342793dd55249089a7289f24efa2e7b] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=37.0 K 2024-04-14T11:17:57,222 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 112120 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:57,222 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/A is initiating minor compaction (all files) 2024-04-14T11:17:57,222 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/A in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:57,222 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/f747662db1a44af4949fb422ff2b9dc3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1fa353e3563848089ae15d60839b2107, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c73db47d0b1a49df84c95a25ef22b7c2] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=109.5 K 2024-04-14T11:17:57,222 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:57,222 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/f747662db1a44af4949fb422ff2b9dc3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1fa353e3563848089ae15d60839b2107, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c73db47d0b1a49df84c95a25ef22b7c2] 2024-04-14T11:17:57,222 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting d344a462af5a442695a2db2d03c7f4fd, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1713093472978 2024-04-14T11:17:57,222 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting f747662db1a44af4949fb422ff2b9dc3, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1713093472978 2024-04-14T11:17:57,223 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 17458b397779414b8b578282b3905b10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1713093474094 2024-04-14T11:17:57,223 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1fa353e3563848089ae15d60839b2107, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1713093474094 2024-04-14T11:17:57,225 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 7342793dd55249089a7289f24efa2e7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1713093475236 2024-04-14T11:17:57,225 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c73db47d0b1a49df84c95a25ef22b7c2, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1713093475236 2024-04-14T11:17:57,233 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#B#compaction#645 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:57,234 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/8491d9117a0f4c979e69dde892bb6817 is 50, key is test_row_0/B:col10/1713093476349/Put/seqid=0 2024-04-14T11:17:57,234 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:57,236 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20240414634b57ea1a714203b117b6ae615f8ba0_d2b4a72b9f8ab01eabcbb9f313e8e868 store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:57,238 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20240414634b57ea1a714203b117b6ae615f8ba0_d2b4a72b9f8ab01eabcbb9f313e8e868, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:57,238 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414634b57ea1a714203b117b6ae615f8ba0_d2b4a72b9f8ab01eabcbb9f313e8e868 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:57,244 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#A#compaction#646 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:57,245 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/80883fd8a13949e89162507aab485094 is 175, key is test_row_0/A:col10/1713093476349/Put/seqid=0 2024-04-14T11:17:57,254 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/80883fd8a13949e89162507aab485094 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/80883fd8a13949e89162507aab485094 2024-04-14T11:17:57,260 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/A of d2b4a72b9f8ab01eabcbb9f313e8e868 into 80883fd8a13949e89162507aab485094(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:57,260 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:57,260 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/A, priority=13, startTime=1713093477220; duration=0sec 2024-04-14T11:17:57,260 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:57,260 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:A 2024-04-14T11:17:57,260 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:57,261 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37840 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:57,261 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/C is initiating minor compaction (all files) 2024-04-14T11:17:57,262 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/C in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:57,262 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c2a96eeb8c5543c1aad7361f653dc87e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4076bdec7c6045a49eed3d3e870dd4ea, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c59fc433a92d43f7a1ad84b085a60d21] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=37.0 K 2024-04-14T11:17:57,262 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c2a96eeb8c5543c1aad7361f653dc87e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1713093472978 2024-04-14T11:17:57,262 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 4076bdec7c6045a49eed3d3e870dd4ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1713093474094 2024-04-14T11:17:57,263 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting c59fc433a92d43f7a1ad84b085a60d21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1713093475236 2024-04-14T11:17:57,270 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#C#compaction#647 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:57,270 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/58aa40fbfa5e456d81c3eaaff56d6754 is 50, key is test_row_0/C:col10/1713093476349/Put/seqid=0 2024-04-14T11:17:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=190 2024-04-14T11:17:57,281 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/58aa40fbfa5e456d81c3eaaff56d6754 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/58aa40fbfa5e456d81c3eaaff56d6754 2024-04-14T11:17:57,284 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:57,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=191 2024-04-14T11:17:57,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:57,285 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-04-14T11:17:57,291 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/C of d2b4a72b9f8ab01eabcbb9f313e8e868 into 58aa40fbfa5e456d81c3eaaff56d6754(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:57,291 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:57,291 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/C, priority=13, startTime=1713093477220; duration=0sec 2024-04-14T11:17:57,291 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:57,291 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:C 2024-04-14T11:17:57,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404145a423269c5e84a2587e4fa39c90979c0_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093476358/Put/seqid=0 2024-04-14T11:17:57,310 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404145a423269c5e84a2587e4fa39c90979c0_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404145a423269c5e84a2587e4fa39c90979c0_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:57,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/72781540cf87466c978a81329b39e95a, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:57,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/72781540cf87466c978a81329b39e95a is 175, key is test_row_0/A:col10/1713093476358/Put/seqid=0 2024-04-14T11:17:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:57,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:57,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093537436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:57,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093537483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:57,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093537484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:57,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093537488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:57,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093537489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:57,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093537538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:57,645 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/8491d9117a0f4c979e69dde892bb6817 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/8491d9117a0f4c979e69dde892bb6817 2024-04-14T11:17:57,652 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/B of d2b4a72b9f8ab01eabcbb9f313e8e868 into 8491d9117a0f4c979e69dde892bb6817(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:57,652 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:57,652 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/B, priority=13, startTime=1713093477220; duration=0sec 2024-04-14T11:17:57,652 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:57,652 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:B 2024-04-14T11:17:57,718 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=483, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/72781540cf87466c978a81329b39e95a 2024-04-14T11:17:57,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/2f29a0a9011b46438a502f0ef9d8d249 is 50, key is test_row_0/B:col10/1713093476358/Put/seqid=0 2024-04-14T11:17:57,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:57,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093537742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:57,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=190 2024-04-14T11:17:58,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:58,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093538044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:58,130 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/2f29a0a9011b46438a502f0ef9d8d249 2024-04-14T11:17:58,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/32c8c5de4cea458fabae2610e0ab40c5 is 50, key is test_row_0/C:col10/1713093476358/Put/seqid=0 2024-04-14T11:17:58,142 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/32c8c5de4cea458fabae2610e0ab40c5 2024-04-14T11:17:58,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/72781540cf87466c978a81329b39e95a as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/72781540cf87466c978a81329b39e95a 2024-04-14T11:17:58,153 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/72781540cf87466c978a81329b39e95a, entries=150, sequenceid=483, filesize=30.5 K 2024-04-14T11:17:58,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/2f29a0a9011b46438a502f0ef9d8d249 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/2f29a0a9011b46438a502f0ef9d8d249 2024-04-14T11:17:58,159 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/2f29a0a9011b46438a502f0ef9d8d249, entries=150, sequenceid=483, filesize=12.0 K 2024-04-14T11:17:58,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/32c8c5de4cea458fabae2610e0ab40c5 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/32c8c5de4cea458fabae2610e0ab40c5 2024-04-14T11:17:58,165 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/32c8c5de4cea458fabae2610e0ab40c5, entries=150, sequenceid=483, filesize=12.0 K 2024-04-14T11:17:58,166 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegion(3035): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 880ms, sequenceid=483, compaction requested=false 2024-04-14T11:17:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=191}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=191 2024-04-14T11:17:58,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=191 2024-04-14T11:17:58,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=191, resume processing ppid=190 2024-04-14T11:17:58,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=191, ppid=190, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4930 sec 2024-04-14T11:17:58,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=190, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=190, table=TestAcidGuarantees in 1.4980 sec 2024-04-14T11:17:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:58,489 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:58,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-04-14T11:17:58,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404143aec2138fc794bd0ab8a64bdf20f0406_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093477425/Put/seqid=0 2024-04-14T11:17:58,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093538496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:58,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:58,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093538498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:58,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:58,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093538499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:58,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:58,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093538499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:58,504 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404143aec2138fc794bd0ab8a64bdf20f0406_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404143aec2138fc794bd0ab8a64bdf20f0406_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:58,505 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/bdbd45830e3d4092912d7e97f25a675f, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:58,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/bdbd45830e3d4092912d7e97f25a675f is 175, key is test_row_0/A:col10/1713093477425/Put/seqid=0 2024-04-14T11:17:58,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093538550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:58,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:58,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093538601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:58,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093538602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:58,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:58,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093538603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=190 2024-04-14T11:17:58,779 INFO [Thread-2769 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 190 completed 2024-04-14T11:17:58,780 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=192, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=192, table=TestAcidGuarantees 2024-04-14T11:17:58,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=192 2024-04-14T11:17:58,783 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=192, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=192, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:58,784 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=192, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=192, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:58,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=193, ppid=192, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:58,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093538804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:58,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093538805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:58,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093538806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=192 2024-04-14T11:17:58,911 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=513, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/bdbd45830e3d4092912d7e97f25a675f 2024-04-14T11:17:58,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/9ec0454ab7c24399b4303fe9a7a36d57 is 50, key is test_row_0/B:col10/1713093477425/Put/seqid=0 2024-04-14T11:17:58,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/9ec0454ab7c24399b4303fe9a7a36d57 2024-04-14T11:17:58,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/f9a9dae2cb944d55acd204b846067b6a is 50, key is test_row_0/C:col10/1713093477425/Put/seqid=0 2024-04-14T11:17:58,936 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:58,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=193 2024-04-14T11:17:58,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:58,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:58,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:58,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] handler.RSProcedureHandler(53): pid=193 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:58,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=193 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:58,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/f9a9dae2cb944d55acd204b846067b6a 2024-04-14T11:17:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=193 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:17:58,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/bdbd45830e3d4092912d7e97f25a675f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/bdbd45830e3d4092912d7e97f25a675f 2024-04-14T11:17:58,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/bdbd45830e3d4092912d7e97f25a675f, entries=150, sequenceid=513, filesize=30.5 K 2024-04-14T11:17:58,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/9ec0454ab7c24399b4303fe9a7a36d57 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/9ec0454ab7c24399b4303fe9a7a36d57 2024-04-14T11:17:58,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/9ec0454ab7c24399b4303fe9a7a36d57, entries=150, sequenceid=513, filesize=12.0 K 2024-04-14T11:17:58,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/f9a9dae2cb944d55acd204b846067b6a as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/f9a9dae2cb944d55acd204b846067b6a 2024-04-14T11:17:58,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/f9a9dae2cb944d55acd204b846067b6a, entries=150, sequenceid=513, filesize=12.0 K 2024-04-14T11:17:58,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 474ms, sequenceid=513, compaction requested=true 2024-04-14T11:17:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:17:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:58,964 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:17:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:17:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:58,964 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:58,965 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94838 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:58,965 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/A is initiating minor compaction (all files) 2024-04-14T11:17:58,965 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37976 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:58,965 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/A in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:58,965 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/B is initiating minor compaction (all files) 2024-04-14T11:17:58,965 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/B in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:58,965 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/80883fd8a13949e89162507aab485094, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/72781540cf87466c978a81329b39e95a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/bdbd45830e3d4092912d7e97f25a675f] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=92.6 K 2024-04-14T11:17:58,966 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/8491d9117a0f4c979e69dde892bb6817, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/2f29a0a9011b46438a502f0ef9d8d249, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/9ec0454ab7c24399b4303fe9a7a36d57] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=37.1 K 2024-04-14T11:17:58,966 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:58,966 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/80883fd8a13949e89162507aab485094, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/72781540cf87466c978a81329b39e95a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/bdbd45830e3d4092912d7e97f25a675f] 2024-04-14T11:17:58,966 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 8491d9117a0f4c979e69dde892bb6817, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1713093475236 2024-04-14T11:17:58,966 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 80883fd8a13949e89162507aab485094, keycount=150, bloomtype=ROW, size=31.6 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1713093475236 2024-04-14T11:17:58,966 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 2f29a0a9011b46438a502f0ef9d8d249, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1713093476353 2024-04-14T11:17:58,967 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 72781540cf87466c978a81329b39e95a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1713093476353 2024-04-14T11:17:58,967 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 9ec0454ab7c24399b4303fe9a7a36d57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1713093477425 2024-04-14T11:17:58,967 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting bdbd45830e3d4092912d7e97f25a675f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1713093477425 2024-04-14T11:17:58,975 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:58,977 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#B#compaction#654 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:58,977 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202404143847dd71b3ee4c448a625c1d7f1e1747_d2b4a72b9f8ab01eabcbb9f313e8e868 store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:58,977 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/4e1506d0acf048ba8000124b3a188bdd is 50, key is test_row_0/B:col10/1713093477425/Put/seqid=0 2024-04-14T11:17:58,978 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202404143847dd71b3ee4c448a625c1d7f1e1747_d2b4a72b9f8ab01eabcbb9f313e8e868, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:58,978 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404143847dd71b3ee4c448a625c1d7f1e1747_d2b4a72b9f8ab01eabcbb9f313e8e868 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:58,988 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#A#compaction#655 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:58,988 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/83fb6f57d1b341f18f293a83ad21cee7 is 175, key is test_row_0/A:col10/1713093477425/Put/seqid=0 2024-04-14T11:17:58,990 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/4e1506d0acf048ba8000124b3a188bdd as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/4e1506d0acf048ba8000124b3a188bdd 2024-04-14T11:17:58,998 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/B of d2b4a72b9f8ab01eabcbb9f313e8e868 into 4e1506d0acf048ba8000124b3a188bdd(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:58,998 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:58,998 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/B, priority=13, startTime=1713093478964; duration=0sec 2024-04-14T11:17:58,998 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:17:58,999 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:B 2024-04-14T11:17:58,999 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:17:59,000 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37942 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:17:59,000 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/C is initiating minor compaction (all files) 2024-04-14T11:17:59,000 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/C in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:59,000 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/58aa40fbfa5e456d81c3eaaff56d6754, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/32c8c5de4cea458fabae2610e0ab40c5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/f9a9dae2cb944d55acd204b846067b6a] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=37.1 K 2024-04-14T11:17:59,000 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/83fb6f57d1b341f18f293a83ad21cee7 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/83fb6f57d1b341f18f293a83ad21cee7 2024-04-14T11:17:59,001 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 58aa40fbfa5e456d81c3eaaff56d6754, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1713093475236 2024-04-14T11:17:59,001 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 32c8c5de4cea458fabae2610e0ab40c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1713093476353 2024-04-14T11:17:59,002 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting f9a9dae2cb944d55acd204b846067b6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1713093477425 2024-04-14T11:17:59,008 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/A of d2b4a72b9f8ab01eabcbb9f313e8e868 into 83fb6f57d1b341f18f293a83ad21cee7(size=31.7 K), total size for store is 31.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:59,008 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:59,008 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/A, priority=13, startTime=1713093478964; duration=0sec 2024-04-14T11:17:59,008 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:59,008 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:A 2024-04-14T11:17:59,010 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#C#compaction#656 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:17:59,010 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/dcd91a587ddf49ac82eb42c71b1ddfa7 is 50, key is test_row_0/C:col10/1713093477425/Put/seqid=0 2024-04-14T11:17:59,023 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/dcd91a587ddf49ac82eb42c71b1ddfa7 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/dcd91a587ddf49ac82eb42c71b1ddfa7 2024-04-14T11:17:59,030 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/C of d2b4a72b9f8ab01eabcbb9f313e8e868 into dcd91a587ddf49ac82eb42c71b1ddfa7(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:17:59,030 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:59,030 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/C, priority=13, startTime=1713093478964; duration=0sec 2024-04-14T11:17:59,030 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:17:59,030 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:C 2024-04-14T11:17:59,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=192 2024-04-14T11:17:59,089 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:59,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=193 2024-04-14T11:17:59,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:59,090 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-04-14T11:17:59,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414bddd362a7a1f466aba52d39fb92e8869_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093478492/Put/seqid=0 2024-04-14T11:17:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:59,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:17:59,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:59,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093539133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:59,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:59,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093539134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:59,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:59,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093539135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:59,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093539236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:59,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093539237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:59,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093539238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=192 2024-04-14T11:17:59,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093539446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:59,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:59,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093539446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093539446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:59,507 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414bddd362a7a1f466aba52d39fb92e8869_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414bddd362a7a1f466aba52d39fb92e8869_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:59,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/19b9eea7f53542f995bbd66bbe2101bd, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:59,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/19b9eea7f53542f995bbd66bbe2101bd is 175, key is test_row_0/A:col10/1713093478492/Put/seqid=0 2024-04-14T11:17:59,514 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=525, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/19b9eea7f53542f995bbd66bbe2101bd 2024-04-14T11:17:59,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/91b5918f010c42519f390b54649d4eb9 is 50, key is test_row_0/B:col10/1713093478492/Put/seqid=0 2024-04-14T11:17:59,526 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/91b5918f010c42519f390b54649d4eb9 2024-04-14T11:17:59,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/4a86759cace4447784bd83883e68c86c is 50, key is test_row_0/C:col10/1713093478492/Put/seqid=0 2024-04-14T11:17:59,537 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/4a86759cace4447784bd83883e68c86c 2024-04-14T11:17:59,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/19b9eea7f53542f995bbd66bbe2101bd as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/19b9eea7f53542f995bbd66bbe2101bd 2024-04-14T11:17:59,547 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/19b9eea7f53542f995bbd66bbe2101bd, entries=150, sequenceid=525, filesize=30.5 K 2024-04-14T11:17:59,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.StoreScanner(974): StoreScanner already has the close lock. There is no need to updateReaders 2024-04-14T11:17:59,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/91b5918f010c42519f390b54649d4eb9 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/91b5918f010c42519f390b54649d4eb9 2024-04-14T11:17:59,554 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/91b5918f010c42519f390b54649d4eb9, entries=150, sequenceid=525, filesize=12.0 K 2024-04-14T11:17:59,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/4a86759cace4447784bd83883e68c86c as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4a86759cace4447784bd83883e68c86c 2024-04-14T11:17:59,561 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4a86759cace4447784bd83883e68c86c, entries=150, sequenceid=525, filesize=12.0 K 2024-04-14T11:17:59,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093539561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:59,562 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(3035): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 472ms, sequenceid=525, compaction requested=false 2024-04-14T11:17:59,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:17:59,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:17:59,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=193}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=193 2024-04-14T11:17:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=193 2024-04-14T11:17:59,564 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=193, resume processing ppid=192 2024-04-14T11:17:59,564 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=193, ppid=192, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 779 msec 2024-04-14T11:17:59,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=192, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=192, table=TestAcidGuarantees in 785 msec 2024-04-14T11:17:59,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:59,750 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:17:59,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-04-14T11:17:59,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093539757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:59,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093539757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:59,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:59,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093539758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:59,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414e9c2cf0b1d694b50851e0027bedfe39c_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093479749/Put/seqid=0 2024-04-14T11:17:59,780 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414e9c2cf0b1d694b50851e0027bedfe39c_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414e9c2cf0b1d694b50851e0027bedfe39c_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:17:59,781 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/30cb7fcfd936498f99c2ec0f07308eed, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:17:59,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/30cb7fcfd936498f99c2ec0f07308eed is 175, key is test_row_0/A:col10/1713093479749/Put/seqid=0 2024-04-14T11:17:59,787 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=555, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/30cb7fcfd936498f99c2ec0f07308eed 2024-04-14T11:17:59,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/3f74b443909240d0aa2156110fde8f2a is 50, key is test_row_0/B:col10/1713093479749/Put/seqid=0 2024-04-14T11:17:59,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093539861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:59,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093539861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:59,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:17:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093539861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:17:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=192 2024-04-14T11:17:59,888 INFO [Thread-2769 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 192 completed 2024-04-14T11:17:59,889 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$22(4374): Client=jenkins//172.31.2.20 flush TestAcidGuarantees 2024-04-14T11:17:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=194, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=194, table=TestAcidGuarantees 2024-04-14T11:17:59,892 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=194, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=194, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-04-14T11:17:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=194 2024-04-14T11:17:59,893 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(88): pid=194, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=194, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-04-14T11:17:59,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=195, ppid=194, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-04-14T11:17:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=194 2024-04-14T11:18:00,044 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:00,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=195 2024-04-14T11:18:00,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:00,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:18:00,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:00,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] handler.RSProcedureHandler(53): pid=195 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:18:00,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=195 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:18:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=195 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:18:00,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093540063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:00,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093540063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:00,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093540064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=194 2024-04-14T11:18:00,197 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:00,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=195 2024-04-14T11:18:00,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:00,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:18:00,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:00,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] handler.RSProcedureHandler(53): pid=195 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:18:00,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=195 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:18:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=195 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:18:00,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/3f74b443909240d0aa2156110fde8f2a 2024-04-14T11:18:00,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/f75e51882dff4ac7b0608ac531f4cf7e is 50, key is test_row_0/C:col10/1713093479749/Put/seqid=0 2024-04-14T11:18:00,350 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:00,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=195 2024-04-14T11:18:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:18:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:00,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=195}] handler.RSProcedureHandler(53): pid=195 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:18:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=195 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:18:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=195 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:18:00,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093540366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:00,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093540367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:00,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093540368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:00,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=194 2024-04-14T11:18:00,503 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:00,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=195 2024-04-14T11:18:00,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:00,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:18:00,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:00,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] handler.RSProcedureHandler(53): pid=195 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:18:00,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.RemoteProcedureResultReporter(57): Failed to complete execution of pid=195 java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:18:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4102): Remote procedure failed, pid=195 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2555) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1259) ~[?:1.8.0_362] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1082) ~[?:1.8.0_362] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2550) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17358) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:51) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:18:00,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:00,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46162 deadline: 1713093540518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:00,521 DEBUG [Thread-2761 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4146 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., hostname=jenkins-hbase6.apache.org,33239,1713093321027, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2024-04-14T11:18:00,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/f75e51882dff4ac7b0608ac531f4cf7e 2024-04-14T11:18:00,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/30cb7fcfd936498f99c2ec0f07308eed as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/30cb7fcfd936498f99c2ec0f07308eed 2024-04-14T11:18:00,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/30cb7fcfd936498f99c2ec0f07308eed, entries=150, sequenceid=555, filesize=30.5 K 2024-04-14T11:18:00,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/3f74b443909240d0aa2156110fde8f2a as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/3f74b443909240d0aa2156110fde8f2a 2024-04-14T11:18:00,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/3f74b443909240d0aa2156110fde8f2a, entries=150, sequenceid=555, filesize=12.0 K 2024-04-14T11:18:00,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/f75e51882dff4ac7b0608ac531f4cf7e as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/f75e51882dff4ac7b0608ac531f4cf7e 2024-04-14T11:18:00,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/f75e51882dff4ac7b0608ac531f4cf7e, entries=150, sequenceid=555, filesize=12.0 K 2024-04-14T11:18:00,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 889ms, sequenceid=555, compaction requested=true 2024-04-14T11:18:00,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:18:00,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:18:00,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:18:00,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:18:00,640 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:18:00,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:18:00,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:18:00,640 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:18:00,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:18:00,641 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94940 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:18:00,641 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/A is initiating minor compaction (all files) 2024-04-14T11:18:00,641 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/A in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:00,641 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/83fb6f57d1b341f18f293a83ad21cee7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/19b9eea7f53542f995bbd66bbe2101bd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/30cb7fcfd936498f99c2ec0f07308eed] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=92.7 K 2024-04-14T11:18:00,641 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38078 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:18:00,641 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:00,641 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/B is initiating minor compaction (all files) 2024-04-14T11:18:00,641 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/83fb6f57d1b341f18f293a83ad21cee7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/19b9eea7f53542f995bbd66bbe2101bd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/30cb7fcfd936498f99c2ec0f07308eed] 2024-04-14T11:18:00,641 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/B in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:00,641 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/4e1506d0acf048ba8000124b3a188bdd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/91b5918f010c42519f390b54649d4eb9, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/3f74b443909240d0aa2156110fde8f2a] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=37.2 K 2024-04-14T11:18:00,642 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 83fb6f57d1b341f18f293a83ad21cee7, keycount=150, bloomtype=ROW, size=31.7 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1713093477425 2024-04-14T11:18:00,642 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 4e1506d0acf048ba8000124b3a188bdd, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1713093477425 2024-04-14T11:18:00,642 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 19b9eea7f53542f995bbd66bbe2101bd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1713093478490 2024-04-14T11:18:00,642 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 91b5918f010c42519f390b54649d4eb9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1713093478490 2024-04-14T11:18:00,643 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 30cb7fcfd936498f99c2ec0f07308eed, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1713093479130 2024-04-14T11:18:00,643 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 3f74b443909240d0aa2156110fde8f2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1713093479130 2024-04-14T11:18:00,650 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:18:00,651 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#B#compaction#664 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:18:00,652 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/9f8837ebcfd64a6c96c29b2be41ee4d2 is 50, key is test_row_0/B:col10/1713093479749/Put/seqid=0 2024-04-14T11:18:00,652 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202404141b51f7bfc7444242822307d4c2a444ea_d2b4a72b9f8ab01eabcbb9f313e8e868 store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:18:00,654 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202404141b51f7bfc7444242822307d4c2a444ea_d2b4a72b9f8ab01eabcbb9f313e8e868, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:18:00,654 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202404141b51f7bfc7444242822307d4c2a444ea_d2b4a72b9f8ab01eabcbb9f313e8e868 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:18:00,656 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:00,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33239 {}] regionserver.RSRpcServices(3971): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=195 2024-04-14T11:18:00,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:00,657 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-04-14T11:18:00,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414dd3b0c8ff62d4b74b96ab14da7c75478_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_1/A:col10/1713093479752/Put/seqid=0 2024-04-14T11:18:00,686 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#A#compaction#663 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:18:00,687 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/ec201140bd5c4a1d881562e6447c4d95 is 175, key is test_row_0/A:col10/1713093479749/Put/seqid=0 2024-04-14T11:18:00,690 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/9f8837ebcfd64a6c96c29b2be41ee4d2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/9f8837ebcfd64a6c96c29b2be41ee4d2 2024-04-14T11:18:00,697 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/B of d2b4a72b9f8ab01eabcbb9f313e8e868 into 9f8837ebcfd64a6c96c29b2be41ee4d2(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:18:00,697 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:18:00,697 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/B, priority=13, startTime=1713093480640; duration=0sec 2024-04-14T11:18:00,697 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:18:00,697 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:B 2024-04-14T11:18:00,697 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:18:00,698 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/ec201140bd5c4a1d881562e6447c4d95 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ec201140bd5c4a1d881562e6447c4d95 2024-04-14T11:18:00,699 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38044 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:18:00,699 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/C is initiating minor compaction (all files) 2024-04-14T11:18:00,699 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/C in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:00,699 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/dcd91a587ddf49ac82eb42c71b1ddfa7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4a86759cace4447784bd83883e68c86c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/f75e51882dff4ac7b0608ac531f4cf7e] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=37.2 K 2024-04-14T11:18:00,700 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting dcd91a587ddf49ac82eb42c71b1ddfa7, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1713093477425 2024-04-14T11:18:00,700 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 4a86759cace4447784bd83883e68c86c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1713093478490 2024-04-14T11:18:00,701 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting f75e51882dff4ac7b0608ac531f4cf7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1713093479130 2024-04-14T11:18:00,705 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/A of d2b4a72b9f8ab01eabcbb9f313e8e868 into ec201140bd5c4a1d881562e6447c4d95(size=31.8 K), total size for store is 31.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:18:00,705 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:18:00,705 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/A, priority=13, startTime=1713093480639; duration=0sec 2024-04-14T11:18:00,705 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:18:00,705 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:A 2024-04-14T11:18:00,708 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#C#compaction#666 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:18:00,708 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/90a50ea2d9f14369b11de68f22c825b1 is 50, key is test_row_0/C:col10/1713093479749/Put/seqid=0 2024-04-14T11:18:00,718 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/90a50ea2d9f14369b11de68f22c825b1 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/90a50ea2d9f14369b11de68f22c825b1 2024-04-14T11:18:00,724 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/C of d2b4a72b9f8ab01eabcbb9f313e8e868 into 90a50ea2d9f14369b11de68f22c825b1(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:18:00,724 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:18:00,724 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/C, priority=13, startTime=1713093480640; duration=0sec 2024-04-14T11:18:00,724 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:18:00,724 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:C 2024-04-14T11:18:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:00,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2491): NOT flushing TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. as already flushing 2024-04-14T11:18:00,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093540907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:00,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:00,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093540908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:00,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:00,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093540908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:01,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=194 2024-04-14T11:18:01,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093541009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:01,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093541010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:01,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093541011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:01,095 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20240414dd3b0c8ff62d4b74b96ab14da7c75478_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414dd3b0c8ff62d4b74b96ab14da7c75478_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:01,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/1cc96566910f4d1894d57aed9ac68e12, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:18:01,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/1cc96566910f4d1894d57aed9ac68e12 is 175, key is test_row_1/A:col10/1713093479752/Put/seqid=0 2024-04-14T11:18:01,102 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=561, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/1cc96566910f4d1894d57aed9ac68e12 2024-04-14T11:18:01,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/4f56172566714abb9b754ca81dd550a1 is 50, key is test_row_1/B:col10/1713093479752/Put/seqid=0 2024-04-14T11:18:01,130 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/4f56172566714abb9b754ca81dd550a1 2024-04-14T11:18:01,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/461490cc42fa424695cc8c814fbbda4f is 50, key is test_row_1/C:col10/1713093479752/Put/seqid=0 2024-04-14T11:18:01,141 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=561 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/461490cc42fa424695cc8c814fbbda4f 2024-04-14T11:18:01,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/1cc96566910f4d1894d57aed9ac68e12 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1cc96566910f4d1894d57aed9ac68e12 2024-04-14T11:18:01,152 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1cc96566910f4d1894d57aed9ac68e12, entries=100, sequenceid=561, filesize=22.0 K 2024-04-14T11:18:01,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/4f56172566714abb9b754ca81dd550a1 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/4f56172566714abb9b754ca81dd550a1 2024-04-14T11:18:01,157 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/4f56172566714abb9b754ca81dd550a1, entries=100, sequenceid=561, filesize=9.6 K 2024-04-14T11:18:01,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/461490cc42fa424695cc8c814fbbda4f as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/461490cc42fa424695cc8c814fbbda4f 2024-04-14T11:18:01,162 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/461490cc42fa424695cc8c814fbbda4f, entries=100, sequenceid=561, filesize=9.6 K 2024-04-14T11:18:01,163 INFO [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegion(3035): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=187.85 KB/192360 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 506ms, sequenceid=561, compaction requested=false 2024-04-14T11:18:01,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:18:01,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:01,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_FLUSH_REGIONS, pid=195}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=195 2024-04-14T11:18:01,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster(4094): Remote procedure done, pid=195 2024-04-14T11:18:01,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=195, resume processing ppid=194 2024-04-14T11:18:01,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=195, ppid=194, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2710 sec 2024-04-14T11:18:01,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=194, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=194, table=TestAcidGuarantees in 1.2770 sec 2024-04-14T11:18:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(8563): Flush requested on d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:01,213 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-04-14T11:18:01,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=201.27 KB heapSize=528.09 KB 2024-04-14T11:18:01,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:01,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093541213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093541214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:01,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093541214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:01,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041490888184bbfe421eab2bc0e8a059b7cf_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093481213/Put/seqid=0 2024-04-14T11:18:01,229 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041490888184bbfe421eab2bc0e8a059b7cf_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041490888184bbfe421eab2bc0e8a059b7cf_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:01,230 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/058dfee77d164f8fade721fa9518f6e0, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:18:01,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/058dfee77d164f8fade721fa9518f6e0 is 175, key is test_row_0/A:col10/1713093481213/Put/seqid=0 2024-04-14T11:18:01,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093541317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:01,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:01,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093541317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:01,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46200 deadline: 1713093541518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:01,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46186 deadline: 1713093541519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:01,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46214 deadline: 1713093541519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:01,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] regionserver.HRegion(5054): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-04-14T11:18:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33239 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.31.2.20:46172 deadline: 1713093541564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:01,566 DEBUG [Thread-2763 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4130 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., hostname=jenkins-hbase6.apache.org,33239,1713093321027, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5052) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3224) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3217) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3050) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3013) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:45005) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:750) 2024-04-14T11:18:01,635 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=597, memsize=67.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/058dfee77d164f8fade721fa9518f6e0 2024-04-14T11:18:01,642 DEBUG [Thread-2772 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x74251eef to 127.0.0.1:60913 2024-04-14T11:18:01,642 DEBUG [Thread-2772 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:18:01,642 DEBUG [Thread-2774 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x77737ebd to 127.0.0.1:60913 2024-04-14T11:18:01,643 DEBUG [Thread-2774 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:18:01,643 DEBUG [Thread-2778 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x58eccddb to 127.0.0.1:60913 2024-04-14T11:18:01,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/15dcde0cfa2d483ebfb8e8a28264a3c4 is 50, key is test_row_0/B:col10/1713093481213/Put/seqid=0 2024-04-14T11:18:01,644 DEBUG [Thread-2778 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:18:01,644 DEBUG [Thread-2770 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5bbf00f8 to 127.0.0.1:60913 2024-04-14T11:18:01,644 DEBUG [Thread-2770 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:18:01,644 DEBUG [Thread-2776 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x50f4a068 to 127.0.0.1:60913 2024-04-14T11:18:01,644 DEBUG [Thread-2776 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:18:01,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/15dcde0cfa2d483ebfb8e8a28264a3c4 2024-04-14T11:18:01,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/dd827d34a2e4435c9cf80f128109387d is 50, key is test_row_0/C:col10/1713093481213/Put/seqid=0 2024-04-14T11:18:01,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/dd827d34a2e4435c9cf80f128109387d 2024-04-14T11:18:01,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/058dfee77d164f8fade721fa9518f6e0 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/058dfee77d164f8fade721fa9518f6e0 2024-04-14T11:18:01,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/058dfee77d164f8fade721fa9518f6e0, entries=150, sequenceid=597, filesize=30.5 K 2024-04-14T11:18:01,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/15dcde0cfa2d483ebfb8e8a28264a3c4 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/15dcde0cfa2d483ebfb8e8a28264a3c4 2024-04-14T11:18:01,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/15dcde0cfa2d483ebfb8e8a28264a3c4, entries=150, sequenceid=597, filesize=12.0 K 2024-04-14T11:18:01,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/dd827d34a2e4435c9cf80f128109387d as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/dd827d34a2e4435c9cf80f128109387d 2024-04-14T11:18:01,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/dd827d34a2e4435c9cf80f128109387d, entries=150, sequenceid=597, filesize=12.0 K 2024-04-14T11:18:01,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3035): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 464ms, sequenceid=597, compaction requested=true 2024-04-14T11:18:01,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2533): Flush status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:18:01,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:A, priority=-2147483648, current under compaction store size is 1 2024-04-14T11:18:01,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:18:01,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:B, priority=-2147483648, current under compaction store size is 2 2024-04-14T11:18:01,677 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:18:01,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:18:01,677 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:18:01,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2b4a72b9f8ab01eabcbb9f313e8e868:C, priority=-2147483648, current under compaction store size is 3 2024-04-14T11:18:01,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:18:01,678 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 86348 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:18:01,678 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/A is initiating minor compaction (all files) 2024-04-14T11:18:01,678 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/A in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:01,678 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35736 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:18:01,678 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ec201140bd5c4a1d881562e6447c4d95, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1cc96566910f4d1894d57aed9ac68e12, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/058dfee77d164f8fade721fa9518f6e0] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=84.3 K 2024-04-14T11:18:01,678 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/B is initiating minor compaction (all files) 2024-04-14T11:18:01,678 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/B in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:01,678 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:01,678 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. files: [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ec201140bd5c4a1d881562e6447c4d95, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1cc96566910f4d1894d57aed9ac68e12, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/058dfee77d164f8fade721fa9518f6e0] 2024-04-14T11:18:01,678 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/9f8837ebcfd64a6c96c29b2be41ee4d2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/4f56172566714abb9b754ca81dd550a1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/15dcde0cfa2d483ebfb8e8a28264a3c4] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=34.9 K 2024-04-14T11:18:01,679 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 9f8837ebcfd64a6c96c29b2be41ee4d2, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1713093479130 2024-04-14T11:18:01,679 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting ec201140bd5c4a1d881562e6447c4d95, keycount=150, bloomtype=ROW, size=31.8 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1713093479130 2024-04-14T11:18:01,679 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 4f56172566714abb9b754ca81dd550a1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=561, earliestPutTs=1713093479752 2024-04-14T11:18:01,679 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 15dcde0cfa2d483ebfb8e8a28264a3c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1713093480895 2024-04-14T11:18:01,679 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 1cc96566910f4d1894d57aed9ac68e12, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=561, earliestPutTs=1713093479752 2024-04-14T11:18:01,680 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] compactions.Compactor(223): Compacting 058dfee77d164f8fade721fa9518f6e0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1713093480895 2024-04-14T11:18:01,686 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:18:01,686 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#B#compaction#672 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:18:01,686 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/298070a5ef2b4835ab76ab3e569e244b is 50, key is test_row_0/B:col10/1713093481213/Put/seqid=0 2024-04-14T11:18:01,687 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(625): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024041464420f2315d842148be36f1ffbb6df1b_d2b4a72b9f8ab01eabcbb9f313e8e868 store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:18:01,691 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(646): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024041464420f2315d842148be36f1ffbb6df1b_d2b4a72b9f8ab01eabcbb9f313e8e868, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:18:01,691 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(657): Aborting writer for hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041464420f2315d842148be36f1ffbb6df1b_d2b4a72b9f8ab01eabcbb9f313e8e868 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:18:01,696 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/298070a5ef2b4835ab76ab3e569e244b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/298070a5ef2b4835ab76ab3e569e244b 2024-04-14T11:18:01,699 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#A#compaction#673 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:18:01,700 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/fb6c29fbe3144a03a69b99c2a764bdf5 is 175, key is test_row_0/A:col10/1713093481213/Put/seqid=0 2024-04-14T11:18:01,701 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/B of d2b4a72b9f8ab01eabcbb9f313e8e868 into 298070a5ef2b4835ab76ab3e569e244b(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:18:01,701 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:18:01,701 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/B, priority=13, startTime=1713093481677; duration=0sec 2024-04-14T11:18:01,702 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-04-14T11:18:01,702 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:B 2024-04-14T11:18:01,702 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-04-14T11:18:01,703 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35702 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-04-14T11:18:01,703 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1535): d2b4a72b9f8ab01eabcbb9f313e8e868/C is initiating minor compaction (all files) 2024-04-14T11:18:01,703 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2346): Starting compaction of d2b4a72b9f8ab01eabcbb9f313e8e868/C in TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:01,703 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1172): Starting compaction of [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/90a50ea2d9f14369b11de68f22c825b1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/461490cc42fa424695cc8c814fbbda4f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/dd827d34a2e4435c9cf80f128109387d] into tmpdir=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp, totalSize=34.9 K 2024-04-14T11:18:01,703 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 90a50ea2d9f14369b11de68f22c825b1, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1713093479130 2024-04-14T11:18:01,704 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting 461490cc42fa424695cc8c814fbbda4f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=561, earliestPutTs=1713093479752 2024-04-14T11:18:01,704 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] compactions.Compactor(223): Compacting dd827d34a2e4435c9cf80f128109387d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1713093480895 2024-04-14T11:18:01,709 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/fb6c29fbe3144a03a69b99c2a764bdf5 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/fb6c29fbe3144a03a69b99c2a764bdf5 2024-04-14T11:18:01,711 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2b4a72b9f8ab01eabcbb9f313e8e868#C#compaction#674 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-04-14T11:18:01,712 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/656067c317d74372993e3bbc13725551 is 50, key is test_row_0/C:col10/1713093481213/Put/seqid=0 2024-04-14T11:18:01,714 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/A of d2b4a72b9f8ab01eabcbb9f313e8e868 into fb6c29fbe3144a03a69b99c2a764bdf5(size=31.9 K), total size for store is 31.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:18:01,714 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:18:01,714 INFO [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/A, priority=13, startTime=1713093481677; duration=0sec 2024-04-14T11:18:01,714 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:18:01,714 DEBUG [RS:0;jenkins-hbase6:33239-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:A 2024-04-14T11:18:01,719 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/656067c317d74372993e3bbc13725551 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/656067c317d74372993e3bbc13725551 2024-04-14T11:18:01,724 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HStore(1332): Completed compaction of 3 (all) file(s) in d2b4a72b9f8ab01eabcbb9f313e8e868/C of d2b4a72b9f8ab01eabcbb9f313e8e868 into 656067c317d74372993e3bbc13725551(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-04-14T11:18:01,724 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.HRegion(2376): Compaction status journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:18:01,724 INFO [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868., storeName=d2b4a72b9f8ab01eabcbb9f313e8e868/C, priority=13, startTime=1713093481677; duration=0sec 2024-04-14T11:18:01,725 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-04-14T11:18:01,725 DEBUG [RS:0;jenkins-hbase6:33239-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2b4a72b9f8ab01eabcbb9f313e8e868:C 2024-04-14T11:18:01,822 DEBUG [Thread-2767 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x42b49b11 to 127.0.0.1:60913 2024-04-14T11:18:01,823 DEBUG [Thread-2767 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:18:01,824 DEBUG [Thread-2765 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x2a265051 to 127.0.0.1:60913 2024-04-14T11:18:01,824 DEBUG [Thread-2765 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:18:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=194 2024-04-14T11:18:02,003 INFO [Thread-2769 {}] client.HBaseAdmin$TableFuture(3746): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 194 completed 2024-04-14T11:18:02,025 DEBUG [Thread-2759 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x170f16e4 to 127.0.0.1:60913 2024-04-14T11:18:02,025 DEBUG [Thread-2759 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:18:04,555 DEBUG [Thread-2761 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x53f6ff98 to 127.0.0.1:60913 2024-04-14T11:18:04,555 DEBUG [Thread-2761 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:18:05,588 DEBUG [Thread-2763 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3a6302ef to 127.0.0.1:60913 2024-04-14T11:18:05,589 DEBUG [Thread-2763 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:18:05,589 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-04-14T11:18:05,589 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-04-14T11:18:05,589 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-04-14T11:18:05,589 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-04-14T11:18:05,589 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-04-14T11:18:05,589 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-04-14T11:18:05,589 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-04-14T11:18:05,589 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(398): read 6789 2024-04-14T11:18:05,589 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(398): read 6702 2024-04-14T11:18:05,589 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(398): read 6754 2024-04-14T11:18:05,589 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(398): read 6843 2024-04-14T11:18:05,589 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(398): read 6683 2024-04-14T11:18:05,589 INFO [Listener at localhost/34025 {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-04-14T11:18:05,589 INFO [Listener at localhost/34025 {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-04-14T11:18:05,589 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5d97b1f8 to 127.0.0.1:60913 2024-04-14T11:18:05,590 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:18:05,590 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$18(968): Started disable of TestAcidGuarantees 2024-04-14T11:18:05,591 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$13(2744): Client=jenkins//172.31.2.20 disable TestAcidGuarantees 2024-04-14T11:18:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=196, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-04-14T11:18:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=196 2024-04-14T11:18:05,595 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093485595"}]},"ts":"1713093485595"} 2024-04-14T11:18:05,596 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-04-14T11:18:05,599 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set TestAcidGuarantees to state=DISABLING 2024-04-14T11:18:05,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=197, ppid=196, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2b4a72b9f8ab01eabcbb9f313e8e868, UNASSIGN}] 2024-04-14T11:18:05,600 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(729): Took xlock for pid=197, ppid=196, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2b4a72b9f8ab01eabcbb9f313e8e868, UNASSIGN 2024-04-14T11:18:05,601 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=197 updating hbase:meta row=d2b4a72b9f8ab01eabcbb9f313e8e868, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:05,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1801): Initialized subprocedures=[{pid=198, ppid=197, state=RUNNABLE; CloseRegionProcedure d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027}] 2024-04-14T11:18:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=196 2024-04-14T11:18:05,753 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:05,754 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] handler.UnassignRegionHandler(119): Close d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:05,754 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] regionserver.HRegion(1676): Closing d2b4a72b9f8ab01eabcbb9f313e8e868, disabling compactions & flushes 2024-04-14T11:18:05,754 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] regionserver.HRegion(1698): Closing region TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:05,754 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] regionserver.HRegion(1719): Waiting without time limit for close lock on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:05,754 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] regionserver.HRegion(1786): Acquired close lock on TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. after waiting 0 ms 2024-04-14T11:18:05,754 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] regionserver.HRegion(1796): Updates disabled for region TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:05,754 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] regionserver.HRegion(2832): Flushing d2b4a72b9f8ab01eabcbb9f313e8e868 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-04-14T11:18:05,761 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041436c451b3508d4d228d0332219aabbdf3_d2b4a72b9f8ab01eabcbb9f313e8e868 is 50, key is test_row_0/A:col10/1713093485587/Put/seqid=0 2024-04-14T11:18:05,770 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] regionserver.HMobStore(266): FLUSH Renaming flushed file from hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024041436c451b3508d4d228d0332219aabbdf3_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041436c451b3508d4d228d0332219aabbdf3_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:05,773 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/a237f131e06348168399cd0ad237c0ab, store: [table=TestAcidGuarantees family=A region=d2b4a72b9f8ab01eabcbb9f313e8e868] 2024-04-14T11:18:05,773 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/a237f131e06348168399cd0ad237c0ab is 175, key is test_row_0/A:col10/1713093485587/Put/seqid=0 2024-04-14T11:18:05,777 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=608, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/a237f131e06348168399cd0ad237c0ab 2024-04-14T11:18:05,783 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/72bc3449496449fb956f302698d2cff2 is 50, key is test_row_0/B:col10/1713093485587/Put/seqid=0 2024-04-14T11:18:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=196 2024-04-14T11:18:06,188 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/72bc3449496449fb956f302698d2cff2 2024-04-14T11:18:06,194 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/31185f3ec69442b7908e43224864190b is 50, key is test_row_0/C:col10/1713093485587/Put/seqid=0 2024-04-14T11:18:06,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=196 2024-04-14T11:18:06,599 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=608 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/31185f3ec69442b7908e43224864190b 2024-04-14T11:18:06,604 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/A/a237f131e06348168399cd0ad237c0ab as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/a237f131e06348168399cd0ad237c0ab 2024-04-14T11:18:06,608 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/a237f131e06348168399cd0ad237c0ab, entries=150, sequenceid=608, filesize=30.5 K 2024-04-14T11:18:06,609 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/B/72bc3449496449fb956f302698d2cff2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/72bc3449496449fb956f302698d2cff2 2024-04-14T11:18:06,613 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/72bc3449496449fb956f302698d2cff2, entries=150, sequenceid=608, filesize=12.0 K 2024-04-14T11:18:06,614 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/.tmp/C/31185f3ec69442b7908e43224864190b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/31185f3ec69442b7908e43224864190b 2024-04-14T11:18:06,618 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/31185f3ec69442b7908e43224864190b, entries=150, sequenceid=608, filesize=12.0 K 2024-04-14T11:18:06,619 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] regionserver.HRegion(3035): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for d2b4a72b9f8ab01eabcbb9f313e8e868 in 865ms, sequenceid=608, compaction requested=false 2024-04-14T11:18:06,619 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ffb0ad2f549c4c75824d0573df1681dd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/be64bfb05df04348881d6e9019b5827e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/53ddedb94ccd4b228cb44580c0f320f5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1b0698db581140c185db13119148363a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/14a1ab5a9a4e436b9e8f9921f19bd68e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/fcdd68760320412dbec96d1f8632eea7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c47f044041e740cca2539c9de1b45f85, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/68b045588af44e259f1d3504f94ec95d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/3e8804708dd147f1b78873a0935f8b5e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/25470faa926844f993d3ecec6f4b1d73, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/d6f8df5a108f4589aaf871e2bb7a3c35, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/fb6737a844d24fe3a62884f3b5f6768d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/442b47066f1d42d2a8441a9192ab1da4, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/b471d7a401884b3fbebbeca9c0b1cc44, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/30dbdc4f44b04eb5aea9dca61b179924, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/22ea9944e5bc44b1a8d62c0bfcfe6a80, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/f24a564099444e8fa29bbcebd8f3daa7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/83bec1e9658d4c5cbd2de8d4292a6142, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ff87bb17db764bec89decb8583f1f2b1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/26a441709e84433e84e3cf08d7ede4a5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/4648a11f0ce9431baddfbc81b060ff24, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/35ef2a04a07b49129b92a2903c6a0b08, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/67c84de37e164a50978d37fc6ca4f66f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/453e3fdbf17f4d87baec86040f3bd464, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c03b42392817475a8e135fe8915396ee, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/7c6f8c6afb1d488fab4e4e664b5f0145, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/36260ca37b9b46febe95528ce4512b59, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/a4857fc536dc4deb83d7ec328e09cf02, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/0fca0158146a41628f60c9778a76dd5c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/389b57bac8ef49cdb2814d2f7f7ed3a3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/f747662db1a44af4949fb422ff2b9dc3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1fa353e3563848089ae15d60839b2107, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c73db47d0b1a49df84c95a25ef22b7c2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/80883fd8a13949e89162507aab485094, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/72781540cf87466c978a81329b39e95a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/83fb6f57d1b341f18f293a83ad21cee7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/bdbd45830e3d4092912d7e97f25a675f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/19b9eea7f53542f995bbd66bbe2101bd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ec201140bd5c4a1d881562e6447c4d95, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/30cb7fcfd936498f99c2ec0f07308eed, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1cc96566910f4d1894d57aed9ac68e12, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/058dfee77d164f8fade721fa9518f6e0] to archive 2024-04-14T11:18:06,620 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:18:06,622 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ffb0ad2f549c4c75824d0573df1681dd to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ffb0ad2f549c4c75824d0573df1681dd 2024-04-14T11:18:06,623 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/be64bfb05df04348881d6e9019b5827e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/be64bfb05df04348881d6e9019b5827e 2024-04-14T11:18:06,625 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/53ddedb94ccd4b228cb44580c0f320f5 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/53ddedb94ccd4b228cb44580c0f320f5 2024-04-14T11:18:06,626 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1b0698db581140c185db13119148363a to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1b0698db581140c185db13119148363a 2024-04-14T11:18:06,627 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/14a1ab5a9a4e436b9e8f9921f19bd68e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/14a1ab5a9a4e436b9e8f9921f19bd68e 2024-04-14T11:18:06,629 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/fcdd68760320412dbec96d1f8632eea7 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/fcdd68760320412dbec96d1f8632eea7 2024-04-14T11:18:06,630 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c47f044041e740cca2539c9de1b45f85 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c47f044041e740cca2539c9de1b45f85 2024-04-14T11:18:06,631 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/68b045588af44e259f1d3504f94ec95d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/68b045588af44e259f1d3504f94ec95d 2024-04-14T11:18:06,632 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/3e8804708dd147f1b78873a0935f8b5e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/3e8804708dd147f1b78873a0935f8b5e 2024-04-14T11:18:06,633 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/25470faa926844f993d3ecec6f4b1d73 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/25470faa926844f993d3ecec6f4b1d73 2024-04-14T11:18:06,634 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/d6f8df5a108f4589aaf871e2bb7a3c35 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/d6f8df5a108f4589aaf871e2bb7a3c35 2024-04-14T11:18:06,635 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/fb6737a844d24fe3a62884f3b5f6768d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/fb6737a844d24fe3a62884f3b5f6768d 2024-04-14T11:18:06,636 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/442b47066f1d42d2a8441a9192ab1da4 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/442b47066f1d42d2a8441a9192ab1da4 2024-04-14T11:18:06,637 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/b471d7a401884b3fbebbeca9c0b1cc44 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/b471d7a401884b3fbebbeca9c0b1cc44 2024-04-14T11:18:06,639 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/30dbdc4f44b04eb5aea9dca61b179924 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/30dbdc4f44b04eb5aea9dca61b179924 2024-04-14T11:18:06,640 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/22ea9944e5bc44b1a8d62c0bfcfe6a80 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/22ea9944e5bc44b1a8d62c0bfcfe6a80 2024-04-14T11:18:06,641 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/f24a564099444e8fa29bbcebd8f3daa7 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/f24a564099444e8fa29bbcebd8f3daa7 2024-04-14T11:18:06,642 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/83bec1e9658d4c5cbd2de8d4292a6142 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/83bec1e9658d4c5cbd2de8d4292a6142 2024-04-14T11:18:06,643 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ff87bb17db764bec89decb8583f1f2b1 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ff87bb17db764bec89decb8583f1f2b1 2024-04-14T11:18:06,644 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/26a441709e84433e84e3cf08d7ede4a5 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/26a441709e84433e84e3cf08d7ede4a5 2024-04-14T11:18:06,645 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/4648a11f0ce9431baddfbc81b060ff24 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/4648a11f0ce9431baddfbc81b060ff24 2024-04-14T11:18:06,646 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/35ef2a04a07b49129b92a2903c6a0b08 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/35ef2a04a07b49129b92a2903c6a0b08 2024-04-14T11:18:06,647 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/67c84de37e164a50978d37fc6ca4f66f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/67c84de37e164a50978d37fc6ca4f66f 2024-04-14T11:18:06,648 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/453e3fdbf17f4d87baec86040f3bd464 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/453e3fdbf17f4d87baec86040f3bd464 2024-04-14T11:18:06,649 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c03b42392817475a8e135fe8915396ee to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c03b42392817475a8e135fe8915396ee 2024-04-14T11:18:06,650 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/7c6f8c6afb1d488fab4e4e664b5f0145 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/7c6f8c6afb1d488fab4e4e664b5f0145 2024-04-14T11:18:06,651 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/36260ca37b9b46febe95528ce4512b59 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/36260ca37b9b46febe95528ce4512b59 2024-04-14T11:18:06,652 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/a4857fc536dc4deb83d7ec328e09cf02 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/a4857fc536dc4deb83d7ec328e09cf02 2024-04-14T11:18:06,653 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/0fca0158146a41628f60c9778a76dd5c to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/0fca0158146a41628f60c9778a76dd5c 2024-04-14T11:18:06,654 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/389b57bac8ef49cdb2814d2f7f7ed3a3 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/389b57bac8ef49cdb2814d2f7f7ed3a3 2024-04-14T11:18:06,655 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/f747662db1a44af4949fb422ff2b9dc3 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/f747662db1a44af4949fb422ff2b9dc3 2024-04-14T11:18:06,656 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1fa353e3563848089ae15d60839b2107 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1fa353e3563848089ae15d60839b2107 2024-04-14T11:18:06,657 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c73db47d0b1a49df84c95a25ef22b7c2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/c73db47d0b1a49df84c95a25ef22b7c2 2024-04-14T11:18:06,658 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/80883fd8a13949e89162507aab485094 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/80883fd8a13949e89162507aab485094 2024-04-14T11:18:06,659 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/72781540cf87466c978a81329b39e95a to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/72781540cf87466c978a81329b39e95a 2024-04-14T11:18:06,660 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/83fb6f57d1b341f18f293a83ad21cee7 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/83fb6f57d1b341f18f293a83ad21cee7 2024-04-14T11:18:06,661 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/bdbd45830e3d4092912d7e97f25a675f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/bdbd45830e3d4092912d7e97f25a675f 2024-04-14T11:18:06,662 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/19b9eea7f53542f995bbd66bbe2101bd to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/19b9eea7f53542f995bbd66bbe2101bd 2024-04-14T11:18:06,663 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ec201140bd5c4a1d881562e6447c4d95 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/ec201140bd5c4a1d881562e6447c4d95 2024-04-14T11:18:06,665 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/30cb7fcfd936498f99c2ec0f07308eed to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/30cb7fcfd936498f99c2ec0f07308eed 2024-04-14T11:18:06,666 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1cc96566910f4d1894d57aed9ac68e12 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/1cc96566910f4d1894d57aed9ac68e12 2024-04-14T11:18:06,667 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/058dfee77d164f8fade721fa9518f6e0 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/058dfee77d164f8fade721fa9518f6e0 2024-04-14T11:18:06,668 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/3ab387b3d4f7457489ed3a2412bd8788, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/1fbafa4a7b504a9796c9550214b95713, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/8a22e2b9515b42d1b89fbdeeae1ae3bb, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/94b10956d92642cea097f113036c6374, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/68b291210f4649bf84fc841400f54765, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/be50c5bebecc4e34b25e34aa10224a1d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/806daf5737674dd296bd7968bd722a6a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/15091b2e5eec4a8396ae9eff79fa0395, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/5bceb60a40774c5986ce6bb00a06b14a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/0db9bc7aab404d78b18880f2eefb21b5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/a8e02afdc3f54d449b19522089bede82, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/87195e87b9f04f4d8a91f6a2bd774a65, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/e3b0d2f09b484b4b8141d88dca3a8f54, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/962a8b751a534bb693a93b885f5416b0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/17ece2d83242494eb2d8b69de47105f6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/f512de4bda1041a585ea1e4837078ddf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/bac96139f3ac42b9922c1ee783f2e526, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/124ae69d25f94e0bb566c62c9c9cbe45, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/103b144b0bf94b3c871ee1101bc48817, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/60bdb660926b4605b0ad195129f852a8, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/eb438822d51049acacb2d606efb09d84, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/cc8f4306770245bf9aa37f3da253b352, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/3560bd0048ff492bb923291e8dd6cfaf, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/c4c7df5b0cee4bfca73df705323a85c5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/7afb7c149ef54305b0b32d5b74791b29, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/03d03e7895204845856ba4aed70141ad, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/8c2436508f6c40b0a5d34d590f5604d6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/63cafd1594dc4ce891e2b47f75b8e98d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/9b0eaee2d0804767a83f108b7f5adf8f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/d344a462af5a442695a2db2d03c7f4fd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/04440446f8a34484bcb381b47c9018a6, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/17458b397779414b8b578282b3905b10, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/8491d9117a0f4c979e69dde892bb6817, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/7342793dd55249089a7289f24efa2e7b, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/2f29a0a9011b46438a502f0ef9d8d249, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/4e1506d0acf048ba8000124b3a188bdd, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/9ec0454ab7c24399b4303fe9a7a36d57, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/91b5918f010c42519f390b54649d4eb9, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/9f8837ebcfd64a6c96c29b2be41ee4d2, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/3f74b443909240d0aa2156110fde8f2a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/4f56172566714abb9b754ca81dd550a1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/15dcde0cfa2d483ebfb8e8a28264a3c4] to archive 2024-04-14T11:18:06,669 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:18:06,671 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/3ab387b3d4f7457489ed3a2412bd8788 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/3ab387b3d4f7457489ed3a2412bd8788 2024-04-14T11:18:06,672 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/1fbafa4a7b504a9796c9550214b95713 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/1fbafa4a7b504a9796c9550214b95713 2024-04-14T11:18:06,673 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/8a22e2b9515b42d1b89fbdeeae1ae3bb to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/8a22e2b9515b42d1b89fbdeeae1ae3bb 2024-04-14T11:18:06,675 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/94b10956d92642cea097f113036c6374 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/94b10956d92642cea097f113036c6374 2024-04-14T11:18:06,676 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/68b291210f4649bf84fc841400f54765 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/68b291210f4649bf84fc841400f54765 2024-04-14T11:18:06,677 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/be50c5bebecc4e34b25e34aa10224a1d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/be50c5bebecc4e34b25e34aa10224a1d 2024-04-14T11:18:06,678 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/806daf5737674dd296bd7968bd722a6a to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/806daf5737674dd296bd7968bd722a6a 2024-04-14T11:18:06,679 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/15091b2e5eec4a8396ae9eff79fa0395 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/15091b2e5eec4a8396ae9eff79fa0395 2024-04-14T11:18:06,680 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/5bceb60a40774c5986ce6bb00a06b14a to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/5bceb60a40774c5986ce6bb00a06b14a 2024-04-14T11:18:06,681 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/0db9bc7aab404d78b18880f2eefb21b5 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/0db9bc7aab404d78b18880f2eefb21b5 2024-04-14T11:18:06,683 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/a8e02afdc3f54d449b19522089bede82 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/a8e02afdc3f54d449b19522089bede82 2024-04-14T11:18:06,684 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/87195e87b9f04f4d8a91f6a2bd774a65 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/87195e87b9f04f4d8a91f6a2bd774a65 2024-04-14T11:18:06,685 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/e3b0d2f09b484b4b8141d88dca3a8f54 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/e3b0d2f09b484b4b8141d88dca3a8f54 2024-04-14T11:18:06,686 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/962a8b751a534bb693a93b885f5416b0 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/962a8b751a534bb693a93b885f5416b0 2024-04-14T11:18:06,687 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/17ece2d83242494eb2d8b69de47105f6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/17ece2d83242494eb2d8b69de47105f6 2024-04-14T11:18:06,688 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/f512de4bda1041a585ea1e4837078ddf to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/f512de4bda1041a585ea1e4837078ddf 2024-04-14T11:18:06,689 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/bac96139f3ac42b9922c1ee783f2e526 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/bac96139f3ac42b9922c1ee783f2e526 2024-04-14T11:18:06,690 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/124ae69d25f94e0bb566c62c9c9cbe45 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/124ae69d25f94e0bb566c62c9c9cbe45 2024-04-14T11:18:06,691 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/103b144b0bf94b3c871ee1101bc48817 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/103b144b0bf94b3c871ee1101bc48817 2024-04-14T11:18:06,692 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/60bdb660926b4605b0ad195129f852a8 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/60bdb660926b4605b0ad195129f852a8 2024-04-14T11:18:06,694 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/eb438822d51049acacb2d606efb09d84 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/eb438822d51049acacb2d606efb09d84 2024-04-14T11:18:06,695 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/cc8f4306770245bf9aa37f3da253b352 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/cc8f4306770245bf9aa37f3da253b352 2024-04-14T11:18:06,696 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/3560bd0048ff492bb923291e8dd6cfaf to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/3560bd0048ff492bb923291e8dd6cfaf 2024-04-14T11:18:06,697 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/c4c7df5b0cee4bfca73df705323a85c5 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/c4c7df5b0cee4bfca73df705323a85c5 2024-04-14T11:18:06,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=196 2024-04-14T11:18:06,699 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/7afb7c149ef54305b0b32d5b74791b29 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/7afb7c149ef54305b0b32d5b74791b29 2024-04-14T11:18:06,700 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/03d03e7895204845856ba4aed70141ad to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/03d03e7895204845856ba4aed70141ad 2024-04-14T11:18:06,701 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/8c2436508f6c40b0a5d34d590f5604d6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/8c2436508f6c40b0a5d34d590f5604d6 2024-04-14T11:18:06,703 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/63cafd1594dc4ce891e2b47f75b8e98d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/63cafd1594dc4ce891e2b47f75b8e98d 2024-04-14T11:18:06,704 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/9b0eaee2d0804767a83f108b7f5adf8f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/9b0eaee2d0804767a83f108b7f5adf8f 2024-04-14T11:18:06,705 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/d344a462af5a442695a2db2d03c7f4fd to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/d344a462af5a442695a2db2d03c7f4fd 2024-04-14T11:18:06,707 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/04440446f8a34484bcb381b47c9018a6 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/04440446f8a34484bcb381b47c9018a6 2024-04-14T11:18:06,708 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/17458b397779414b8b578282b3905b10 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/17458b397779414b8b578282b3905b10 2024-04-14T11:18:06,709 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/8491d9117a0f4c979e69dde892bb6817 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/8491d9117a0f4c979e69dde892bb6817 2024-04-14T11:18:06,710 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/7342793dd55249089a7289f24efa2e7b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/7342793dd55249089a7289f24efa2e7b 2024-04-14T11:18:06,711 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/2f29a0a9011b46438a502f0ef9d8d249 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/2f29a0a9011b46438a502f0ef9d8d249 2024-04-14T11:18:06,712 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/4e1506d0acf048ba8000124b3a188bdd to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/4e1506d0acf048ba8000124b3a188bdd 2024-04-14T11:18:06,713 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/9ec0454ab7c24399b4303fe9a7a36d57 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/9ec0454ab7c24399b4303fe9a7a36d57 2024-04-14T11:18:06,714 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/91b5918f010c42519f390b54649d4eb9 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/91b5918f010c42519f390b54649d4eb9 2024-04-14T11:18:06,715 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/9f8837ebcfd64a6c96c29b2be41ee4d2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/9f8837ebcfd64a6c96c29b2be41ee4d2 2024-04-14T11:18:06,716 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/3f74b443909240d0aa2156110fde8f2a to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/3f74b443909240d0aa2156110fde8f2a 2024-04-14T11:18:06,717 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/4f56172566714abb9b754ca81dd550a1 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/4f56172566714abb9b754ca81dd550a1 2024-04-14T11:18:06,718 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/15dcde0cfa2d483ebfb8e8a28264a3c4 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/15dcde0cfa2d483ebfb8e8a28264a3c4 2024-04-14T11:18:06,719 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/7cf5af8c3dfd455fb98f7f6254953398, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/d0ffcaa00bf04b03ad76de7fcc02197f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/3744c3bede2f4d16a3624a2c0de97469, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/41669d25a0124a90a3f610e066945122, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/bb113cde374c4e4bac4d8aee8895fa62, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c85b40eca80f4d94b85a55d9e851ad93, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/9bfd888a99824005bc547f5ccd7533b3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/6ae13013924b459a82732092d039183c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/0bd04cd553fb469da0145d45e1e67480, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/49baa07997d74ca7a248ad60c15a1317, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/3b27f0036a154f748343c496932d7cf3, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/1cb92373e85a481aa7f165d7a343d7e5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/a13d51ae71ef451bac6fb35d79c86f7a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/a4f0d130c8f940beaa45aae9c95a105f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4ee7ca1e9746479a81850c8554a595d4, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/e37f1c55399c4ef58606270fe7438679, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/7e6e9fbf44c548afaf73ca2d3988b40f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/e70ca28462ae4934a034ad17aeca7d0d, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/22a91df9f9cc4b6388de2f7c0b1cca01, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/5b9cac0f91254ba6aa745adada4b4d68, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4e515c961afb41fd97c9fb9e083606eb, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/1244fb2e29af422cac702c603d341c31, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/279f5ce0f13a4987b896d9e93f84717c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/fc1e2b50dc07453ab6eb24a628339f28, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c3101beb67874ca2b67fb4a6902ced54, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/87ba252b3c334c78b6096eb8a07cb0d0, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/b5ec5c508a65483e82600441be29e114, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/785d7230b3be4c0c8536dffd3e9082fa, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c2a96eeb8c5543c1aad7361f653dc87e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/3626796d5d6e464eb080fd2184a1c4f7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4076bdec7c6045a49eed3d3e870dd4ea, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/58aa40fbfa5e456d81c3eaaff56d6754, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c59fc433a92d43f7a1ad84b085a60d21, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/32c8c5de4cea458fabae2610e0ab40c5, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/dcd91a587ddf49ac82eb42c71b1ddfa7, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/f9a9dae2cb944d55acd204b846067b6a, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4a86759cace4447784bd83883e68c86c, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/90a50ea2d9f14369b11de68f22c825b1, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/f75e51882dff4ac7b0608ac531f4cf7e, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/461490cc42fa424695cc8c814fbbda4f, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/dd827d34a2e4435c9cf80f128109387d] to archive 2024-04-14T11:18:06,720 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-04-14T11:18:06,721 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/7cf5af8c3dfd455fb98f7f6254953398 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/7cf5af8c3dfd455fb98f7f6254953398 2024-04-14T11:18:06,722 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/d0ffcaa00bf04b03ad76de7fcc02197f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/d0ffcaa00bf04b03ad76de7fcc02197f 2024-04-14T11:18:06,723 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/3744c3bede2f4d16a3624a2c0de97469 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/3744c3bede2f4d16a3624a2c0de97469 2024-04-14T11:18:06,724 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/41669d25a0124a90a3f610e066945122 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/41669d25a0124a90a3f610e066945122 2024-04-14T11:18:06,725 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/bb113cde374c4e4bac4d8aee8895fa62 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/bb113cde374c4e4bac4d8aee8895fa62 2024-04-14T11:18:06,726 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c85b40eca80f4d94b85a55d9e851ad93 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c85b40eca80f4d94b85a55d9e851ad93 2024-04-14T11:18:06,727 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/9bfd888a99824005bc547f5ccd7533b3 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/9bfd888a99824005bc547f5ccd7533b3 2024-04-14T11:18:06,729 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/6ae13013924b459a82732092d039183c to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/6ae13013924b459a82732092d039183c 2024-04-14T11:18:06,730 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/0bd04cd553fb469da0145d45e1e67480 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/0bd04cd553fb469da0145d45e1e67480 2024-04-14T11:18:06,731 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/49baa07997d74ca7a248ad60c15a1317 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/49baa07997d74ca7a248ad60c15a1317 2024-04-14T11:18:06,732 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/3b27f0036a154f748343c496932d7cf3 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/3b27f0036a154f748343c496932d7cf3 2024-04-14T11:18:06,733 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/1cb92373e85a481aa7f165d7a343d7e5 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/1cb92373e85a481aa7f165d7a343d7e5 2024-04-14T11:18:06,734 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/a13d51ae71ef451bac6fb35d79c86f7a to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/a13d51ae71ef451bac6fb35d79c86f7a 2024-04-14T11:18:06,735 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/a4f0d130c8f940beaa45aae9c95a105f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/a4f0d130c8f940beaa45aae9c95a105f 2024-04-14T11:18:06,736 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4ee7ca1e9746479a81850c8554a595d4 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4ee7ca1e9746479a81850c8554a595d4 2024-04-14T11:18:06,737 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/e37f1c55399c4ef58606270fe7438679 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/e37f1c55399c4ef58606270fe7438679 2024-04-14T11:18:06,738 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/7e6e9fbf44c548afaf73ca2d3988b40f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/7e6e9fbf44c548afaf73ca2d3988b40f 2024-04-14T11:18:06,739 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/e70ca28462ae4934a034ad17aeca7d0d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/e70ca28462ae4934a034ad17aeca7d0d 2024-04-14T11:18:06,740 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/22a91df9f9cc4b6388de2f7c0b1cca01 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/22a91df9f9cc4b6388de2f7c0b1cca01 2024-04-14T11:18:06,741 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/5b9cac0f91254ba6aa745adada4b4d68 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/5b9cac0f91254ba6aa745adada4b4d68 2024-04-14T11:18:06,742 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4e515c961afb41fd97c9fb9e083606eb to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4e515c961afb41fd97c9fb9e083606eb 2024-04-14T11:18:06,743 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/1244fb2e29af422cac702c603d341c31 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/1244fb2e29af422cac702c603d341c31 2024-04-14T11:18:06,744 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/279f5ce0f13a4987b896d9e93f84717c to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/279f5ce0f13a4987b896d9e93f84717c 2024-04-14T11:18:06,746 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/fc1e2b50dc07453ab6eb24a628339f28 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/fc1e2b50dc07453ab6eb24a628339f28 2024-04-14T11:18:06,747 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c3101beb67874ca2b67fb4a6902ced54 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c3101beb67874ca2b67fb4a6902ced54 2024-04-14T11:18:06,748 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/87ba252b3c334c78b6096eb8a07cb0d0 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/87ba252b3c334c78b6096eb8a07cb0d0 2024-04-14T11:18:06,749 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/b5ec5c508a65483e82600441be29e114 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/b5ec5c508a65483e82600441be29e114 2024-04-14T11:18:06,750 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/785d7230b3be4c0c8536dffd3e9082fa to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/785d7230b3be4c0c8536dffd3e9082fa 2024-04-14T11:18:06,751 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c2a96eeb8c5543c1aad7361f653dc87e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c2a96eeb8c5543c1aad7361f653dc87e 2024-04-14T11:18:06,752 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/3626796d5d6e464eb080fd2184a1c4f7 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/3626796d5d6e464eb080fd2184a1c4f7 2024-04-14T11:18:06,753 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4076bdec7c6045a49eed3d3e870dd4ea to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4076bdec7c6045a49eed3d3e870dd4ea 2024-04-14T11:18:06,754 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/58aa40fbfa5e456d81c3eaaff56d6754 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/58aa40fbfa5e456d81c3eaaff56d6754 2024-04-14T11:18:06,756 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c59fc433a92d43f7a1ad84b085a60d21 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/c59fc433a92d43f7a1ad84b085a60d21 2024-04-14T11:18:06,757 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/32c8c5de4cea458fabae2610e0ab40c5 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/32c8c5de4cea458fabae2610e0ab40c5 2024-04-14T11:18:06,758 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/dcd91a587ddf49ac82eb42c71b1ddfa7 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/dcd91a587ddf49ac82eb42c71b1ddfa7 2024-04-14T11:18:06,759 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/f9a9dae2cb944d55acd204b846067b6a to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/f9a9dae2cb944d55acd204b846067b6a 2024-04-14T11:18:06,760 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4a86759cace4447784bd83883e68c86c to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/4a86759cace4447784bd83883e68c86c 2024-04-14T11:18:06,761 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/90a50ea2d9f14369b11de68f22c825b1 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/90a50ea2d9f14369b11de68f22c825b1 2024-04-14T11:18:06,762 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/f75e51882dff4ac7b0608ac531f4cf7e to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/f75e51882dff4ac7b0608ac531f4cf7e 2024-04-14T11:18:06,763 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/461490cc42fa424695cc8c814fbbda4f to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/461490cc42fa424695cc8c814fbbda4f 2024-04-14T11:18:06,764 DEBUG [StoreCloser-TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/dd827d34a2e4435c9cf80f128109387d to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/dd827d34a2e4435c9cf80f128109387d 2024-04-14T11:18:06,768 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/recovered.edits/611.seqid, newMaxSeqId=611, maxSeqId=4 2024-04-14T11:18:06,770 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] regionserver.HRegion(1917): Closed TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868. 2024-04-14T11:18:06,770 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] regionserver.HRegion(1630): Region close journal for d2b4a72b9f8ab01eabcbb9f313e8e868: 2024-04-14T11:18:06,771 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=198}] handler.UnassignRegionHandler(163): Closed d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:06,771 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=197 updating hbase:meta row=d2b4a72b9f8ab01eabcbb9f313e8e868, regionState=CLOSED 2024-04-14T11:18:06,773 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=198, resume processing ppid=197 2024-04-14T11:18:06,773 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1462): Finished pid=198, ppid=197, state=SUCCESS; CloseRegionProcedure d2b4a72b9f8ab01eabcbb9f313e8e868, server=jenkins-hbase6.apache.org,33239,1713093321027 in 1.1700 sec 2024-04-14T11:18:06,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1958): Finished subprocedure pid=197, resume processing ppid=196 2024-04-14T11:18:06,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1462): Finished pid=197, ppid=196, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2b4a72b9f8ab01eabcbb9f313e8e868, UNASSIGN in 1.1730 sec 2024-04-14T11:18:06,775 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1713093486775"}]},"ts":"1713093486775"} 2024-04-14T11:18:06,776 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-04-14T11:18:06,778 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set TestAcidGuarantees to state=DISABLED 2024-04-14T11:18:06,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1462): Finished pid=196, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.1880 sec 2024-04-14T11:18:07,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=196 2024-04-14T11:18:07,700 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$TableFuture(3746): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 196 completed 2024-04-14T11:18:07,701 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.HMaster$5(2494): Client=jenkins//172.31.2.20 delete TestAcidGuarantees 2024-04-14T11:18:07,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] procedure2.ProcedureExecutor(1080): Stored pid=199, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:18:07,703 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=199, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:18:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=199 2024-04-14T11:18:07,704 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=199, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:18:07,707 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,710 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A, FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B, FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C, FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/recovered.edits] 2024-04-14T11:18:07,714 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/a237f131e06348168399cd0ad237c0ab to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/a237f131e06348168399cd0ad237c0ab 2024-04-14T11:18:07,716 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/fb6c29fbe3144a03a69b99c2a764bdf5 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/A/fb6c29fbe3144a03a69b99c2a764bdf5 2024-04-14T11:18:07,720 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/298070a5ef2b4835ab76ab3e569e244b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/298070a5ef2b4835ab76ab3e569e244b 2024-04-14T11:18:07,721 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/72bc3449496449fb956f302698d2cff2 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/B/72bc3449496449fb956f302698d2cff2 2024-04-14T11:18:07,724 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/31185f3ec69442b7908e43224864190b to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/31185f3ec69442b7908e43224864190b 2024-04-14T11:18:07,725 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/656067c317d74372993e3bbc13725551 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/C/656067c317d74372993e3bbc13725551 2024-04-14T11:18:07,728 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/recovered.edits/611.seqid to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868/recovered.edits/611.seqid 2024-04-14T11:18:07,729 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/default/TestAcidGuarantees/d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,729 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived TestAcidGuarantees regions 2024-04-14T11:18:07,730 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-04-14T11:18:07,730 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-04-14T11:18:07,734 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041401ac02d3670d4c85bbc96cd2134ff58c_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041401ac02d3670d4c85bbc96cd2134ff58c_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,735 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404141e081cccb3204057b4770c78a3c697a6_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404141e081cccb3204057b4770c78a3c697a6_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,736 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404142c0f22d2a0ce448bb519165c0c5961da_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404142c0f22d2a0ce448bb519165c0c5961da_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,738 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041436c451b3508d4d228d0332219aabbdf3_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041436c451b3508d4d228d0332219aabbdf3_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,739 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414396bfc3dca63481c804206c3969fd9c2_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414396bfc3dca63481c804206c3969fd9c2_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,740 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404143aec2138fc794bd0ab8a64bdf20f0406_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404143aec2138fc794bd0ab8a64bdf20f0406_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,742 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404143c95ec4683a14d03a6e37a2bcb68374a_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404143c95ec4683a14d03a6e37a2bcb68374a_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,743 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414499c2d1065e948e183cd5ea6555e2943_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414499c2d1065e948e183cd5ea6555e2943_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,744 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041454d48dd67cbd42c0a920cb1e93dbae82_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041454d48dd67cbd42c0a920cb1e93dbae82_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,746 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041457d46b459be54d9e825f99300ddfe55c_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041457d46b459be54d9e825f99300ddfe55c_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,747 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404145a423269c5e84a2587e4fa39c90979c0_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404145a423269c5e84a2587e4fa39c90979c0_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,748 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404146ef7c45d5de54f699d0f30761279ed6d_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404146ef7c45d5de54f699d0f30761279ed6d_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,750 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404147553171a61f64968b8a8bfb30fa27a70_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404147553171a61f64968b8a8bfb30fa27a70_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,751 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041490888184bbfe421eab2bc0e8a059b7cf_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041490888184bbfe421eab2bc0e8a059b7cf_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,752 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404149468e16c69704b23874b9298c3be20eb_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404149468e16c69704b23874b9298c3be20eb_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,754 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041496c808e76b8143508063801cf8753dc8_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024041496c808e76b8143508063801cf8753dc8_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,755 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404149bba818881994c41a20cb15e73d9b1c0_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404149bba818881994c41a20cb15e73d9b1c0_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,756 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404149f6d2edae6bf4293a86e6a2339316396_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202404149f6d2edae6bf4293a86e6a2339316396_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,758 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414a9059ea6753f4affae20c363f7467f75_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414a9059ea6753f4affae20c363f7467f75_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,759 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414bc0bb01f99dd44f3a497dabd02d43d12_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414bc0bb01f99dd44f3a497dabd02d43d12_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,761 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414bddd362a7a1f466aba52d39fb92e8869_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414bddd362a7a1f466aba52d39fb92e8869_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,762 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414c4cda27c090d4f53a890d5ef92e17f10_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414c4cda27c090d4f53a890d5ef92e17f10_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,764 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414d509c4e8a8f34e548ef0689ef6de8c77_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414d509c4e8a8f34e548ef0689ef6de8c77_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,765 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414dd3b0c8ff62d4b74b96ab14da7c75478_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414dd3b0c8ff62d4b74b96ab14da7c75478_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,767 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414e3f2efc7b076497cb15539ccee06984b_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414e3f2efc7b076497cb15539ccee06984b_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,768 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414e939659a8a47432d856632a4585a2a7b_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414e939659a8a47432d856632a4585a2a7b_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,770 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414e9c2cf0b1d694b50851e0027bedfe39c_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414e9c2cf0b1d694b50851e0027bedfe39c_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,771 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414edc35621a3934cb9b98b93a3378b9427_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414edc35621a3934cb9b98b93a3378b9427_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,772 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414fc745d1180d0466dbe45d5a58c3d1276_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414fc745d1180d0466dbe45d5a58c3d1276_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,774 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414fff4979463204557a5950445f2a5ee0c_d2b4a72b9f8ab01eabcbb9f313e8e868 to hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20240414fff4979463204557a5950445f2a5ee0c_d2b4a72b9f8ab01eabcbb9f313e8e868 2024-04-14T11:18:07,774 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-04-14T11:18:07,777 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=199, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:18:07,779 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-04-14T11:18:07,781 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'TestAcidGuarantees' descriptor. 2024-04-14T11:18:07,782 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=199, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:18:07,782 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'TestAcidGuarantees' from region states. 2024-04-14T11:18:07,783 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1713093487782"}]},"ts":"9223372036854775807"} 2024-04-14T11:18:07,784 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-04-14T11:18:07,785 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => d2b4a72b9f8ab01eabcbb9f313e8e868, NAME => 'TestAcidGuarantees,,1713093460283.d2b4a72b9f8ab01eabcbb9f313e8e868.', STARTKEY => '', ENDKEY => ''}] 2024-04-14T11:18:07,785 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'TestAcidGuarantees' as deleted. 2024-04-14T11:18:07,785 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1713093487785"}]},"ts":"9223372036854775807"} 2024-04-14T11:18:07,787 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-04-14T11:18:07,790 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=199, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-04-14T11:18:07,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1462): Finished pid=199, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 89 msec 2024-04-14T11:18:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33819 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=199 2024-04-14T11:18:07,805 INFO [Listener at localhost/34025 {}] client.HBaseAdmin$TableFuture(3746): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 199 completed 2024-04-14T11:18:07,821 INFO [Listener at localhost/34025 {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=221 (was 220) - Thread LEAK? -, OpenFileDescriptor=519 (was 510) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=470 (was 486), ProcessCount=168 (was 168), AvailableMemoryMB=1435 (was 1473) 2024-04-14T11:18:07,821 INFO [Listener at localhost/34025 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-04-14T11:18:07,822 INFO [Listener at localhost/34025 {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-04-14T11:18:07,822 DEBUG [Listener at localhost/34025 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x302b29d8 to 127.0.0.1:60913 2024-04-14T11:18:07,822 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:18:07,822 DEBUG [Listener at localhost/34025 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-04-14T11:18:07,822 DEBUG [Listener at localhost/34025 {}] util.JVMClusterUtil(257): Found active master hash=209494151, stopped=false 2024-04-14T11:18:07,823 INFO [Listener at localhost/34025 {}] master.ServerManager(987): Cluster shutdown requested of master=jenkins-hbase6.apache.org,33819,1713093319730 2024-04-14T11:18:07,825 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-04-14T11:18:07,825 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-04-14T11:18:07,825 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-04-14T11:18:07,825 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-04-14T11:18:07,825 INFO [Listener at localhost/34025 {}] procedure2.ProcedureExecutor(678): Stopping 2024-04-14T11:18:07,826 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-04-14T11:18:07,826 DEBUG [Listener at localhost/34025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:18:07,826 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-04-14T11:18:07,826 INFO [Listener at localhost/34025 {}] regionserver.HRegionServer(2554): ***** STOPPING region server 'jenkins-hbase6.apache.org,33239,1713093321027' ***** 2024-04-14T11:18:07,826 INFO [Listener at localhost/34025 {}] regionserver.HRegionServer(2568): STOPPED: Shutdown requested 2024-04-14T11:18:07,826 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.HeapMemoryManager(220): Stopping 2024-04-14T11:18:07,827 INFO [RS:0;jenkins-hbase6:33239 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-04-14T11:18:07,827 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-04-14T11:18:07,827 INFO [RS:0;jenkins-hbase6:33239 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-04-14T11:18:07,827 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(3570): Received CLOSE for 23980ec0107c2c55bc841b5f2502d0a1 2024-04-14T11:18:07,827 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(1221): stopping server jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:07,827 DEBUG [RS:0;jenkins-hbase6:33239 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:18:07,827 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-04-14T11:18:07,827 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-04-14T11:18:07,827 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-04-14T11:18:07,827 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(3570): Received CLOSE for 1588230740 2024-04-14T11:18:07,828 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(1596): Waiting on 2 regions to close 2024-04-14T11:18:07,828 DEBUG [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(1600): Online Regions={23980ec0107c2c55bc841b5f2502d0a1=hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1., 1588230740=hbase:meta,,1.1588230740} 2024-04-14T11:18:07,829 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Closing 23980ec0107c2c55bc841b5f2502d0a1, disabling compactions & flushes 2024-04-14T11:18:07,829 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Closing 1588230740, disabling compactions & flushes 2024-04-14T11:18:07,829 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1698): Closing region hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1. 2024-04-14T11:18:07,829 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1698): Closing region hbase:meta,,1.1588230740 2024-04-14T11:18:07,829 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1719): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-04-14T11:18:07,829 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1719): Waiting without time limit for close lock on hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1. 2024-04-14T11:18:07,829 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1786): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-04-14T11:18:07,829 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1786): Acquired close lock on hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1. after waiting 0 ms 2024-04-14T11:18:07,829 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1796): Updates disabled for region hbase:meta,,1.1588230740 2024-04-14T11:18:07,829 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1796): Updates disabled for region hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1. 2024-04-14T11:18:07,829 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2832): Flushing 1588230740 3/3 column families, dataSize=20.92 KB heapSize=36.24 KB 2024-04-14T11:18:07,829 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2832): Flushing 23980ec0107c2c55bc841b5f2502d0a1 1/1 column families, dataSize=78 B heapSize=488 B 2024-04-14T11:18:07,831 DEBUG [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(1626): Waiting on 1588230740, 23980ec0107c2c55bc841b5f2502d0a1 2024-04-14T11:18:07,844 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/namespace/23980ec0107c2c55bc841b5f2502d0a1/.tmp/info/4bbb2ebf3c4743af8314c7e1f0e66c4b is 45, key is default/info:d/1713093324648/Put/seqid=0 2024-04-14T11:18:07,848 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/namespace/23980ec0107c2c55bc841b5f2502d0a1/.tmp/info/4bbb2ebf3c4743af8314c7e1f0e66c4b 2024-04-14T11:18:07,851 INFO [regionserver/jenkins-hbase6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-04-14T11:18:07,851 INFO [regionserver/jenkins-hbase6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-04-14T11:18:07,852 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740/.tmp/info/afe788baae65405d964d4a8951aa6ada is 143, key is hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1./info:regioninfo/1713093324549/Put/seqid=0 2024-04-14T11:18:07,853 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/namespace/23980ec0107c2c55bc841b5f2502d0a1/.tmp/info/4bbb2ebf3c4743af8314c7e1f0e66c4b as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/namespace/23980ec0107c2c55bc841b5f2502d0a1/info/4bbb2ebf3c4743af8314c7e1f0e66c4b 2024-04-14T11:18:07,859 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/namespace/23980ec0107c2c55bc841b5f2502d0a1/info/4bbb2ebf3c4743af8314c7e1f0e66c4b, entries=2, sequenceid=6, filesize=4.9 K 2024-04-14T11:18:07,859 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3035): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 23980ec0107c2c55bc841b5f2502d0a1 in 30ms, sequenceid=6, compaction requested=false 2024-04-14T11:18:07,860 INFO [regionserver/jenkins-hbase6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-04-14T11:18:07,864 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/namespace/23980ec0107c2c55bc841b5f2502d0a1/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-04-14T11:18:07,866 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1917): Closed hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1. 2024-04-14T11:18:07,866 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1630): Region close journal for 23980ec0107c2c55bc841b5f2502d0a1: 2024-04-14T11:18:07,866 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1713093323665.23980ec0107c2c55bc841b5f2502d0a1. 2024-04-14T11:18:08,031 DEBUG [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(1626): Waiting on 1588230740 2024-04-14T11:18:08,231 DEBUG [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(1626): Waiting on 1588230740 2024-04-14T11:18:08,256 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.26 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740/.tmp/info/afe788baae65405d964d4a8951aa6ada 2024-04-14T11:18:08,274 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740/.tmp/rep_barrier/9a128fb10917402794d259dbb4236fe0 is 102, key is TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785./rep_barrier:/1713093350002/DeleteFamily/seqid=0 2024-04-14T11:18:08,279 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740/.tmp/rep_barrier/9a128fb10917402794d259dbb4236fe0 2024-04-14T11:18:08,299 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740/.tmp/table/bb8590674a584094b36ec098afcb8e54 is 96, key is TestAcidGuarantees,,1713093324911.5407acadb5e9c9af78221a8f2d7e6785./table:/1713093350002/DeleteFamily/seqid=0 2024-04-14T11:18:08,432 DEBUG [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(1626): Waiting on 1588230740 2024-04-14T11:18:08,632 DEBUG [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(1626): Waiting on 1588230740 2024-04-14T11:18:08,704 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.08 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740/.tmp/table/bb8590674a584094b36ec098afcb8e54 2024-04-14T11:18:08,709 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740/.tmp/info/afe788baae65405d964d4a8951aa6ada as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740/info/afe788baae65405d964d4a8951aa6ada 2024-04-14T11:18:08,713 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740/info/afe788baae65405d964d4a8951aa6ada, entries=22, sequenceid=93, filesize=7.6 K 2024-04-14T11:18:08,714 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740/.tmp/rep_barrier/9a128fb10917402794d259dbb4236fe0 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740/rep_barrier/9a128fb10917402794d259dbb4236fe0 2024-04-14T11:18:08,719 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740/rep_barrier/9a128fb10917402794d259dbb4236fe0, entries=6, sequenceid=93, filesize=5.9 K 2024-04-14T11:18:08,720 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740/.tmp/table/bb8590674a584094b36ec098afcb8e54 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740/table/bb8590674a584094b36ec098afcb8e54 2024-04-14T11:18:08,724 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740/table/bb8590674a584094b36ec098afcb8e54, entries=9, sequenceid=93, filesize=5.8 K 2024-04-14T11:18:08,725 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3035): Finished flush of dataSize ~20.92 KB/21417, heapSize ~36.20 KB/37064, currentSize=0 B/0 for 1588230740 in 895ms, sequenceid=93, compaction requested=false 2024-04-14T11:18:08,730 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/data/hbase/meta/1588230740/recovered.edits/96.seqid, newMaxSeqId=96, maxSeqId=1 2024-04-14T11:18:08,730 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-04-14T11:18:08,731 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1917): Closed hbase:meta,,1.1588230740 2024-04-14T11:18:08,731 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1630): Region close journal for 1588230740: 2024-04-14T11:18:08,731 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-04-14T11:18:08,832 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(1247): stopping server jenkins-hbase6.apache.org,33239,1713093321027; all regions closed. 2024-04-14T11:18:08,841 DEBUG [RS:0;jenkins-hbase6:33239 {}] wal.AbstractFSWAL(1068): Moved 1 WAL file(s) to /user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/oldWALs 2024-04-14T11:18:08,841 INFO [RS:0;jenkins-hbase6:33239 {}] wal.AbstractFSWAL(1071): Closed WAL: AsyncFSWAL jenkins-hbase6.apache.org%2C33239%2C1713093321027.meta:.meta(num 1713093323389) 2024-04-14T11:18:08,846 DEBUG [RS:0;jenkins-hbase6:33239 {}] wal.AbstractFSWAL(1068): Moved 1 WAL file(s) to /user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/oldWALs 2024-04-14T11:18:08,846 INFO [RS:0;jenkins-hbase6:33239 {}] wal.AbstractFSWAL(1071): Closed WAL: AsyncFSWAL jenkins-hbase6.apache.org%2C33239%2C1713093321027:(num 1713093323006) 2024-04-14T11:18:08,846 DEBUG [RS:0;jenkins-hbase6:33239 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:18:08,846 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.LeaseManager(133): Closed leases 2024-04-14T11:18:08,847 INFO [RS:0;jenkins-hbase6:33239 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase6:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-04-14T11:18:08,847 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-04-14T11:18:08,848 INFO [RS:0;jenkins-hbase6:33239 {}] ipc.NettyRpcServer(352): Stopping server on /172.31.2.20:33239 2024-04-14T11:18:08,855 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/jenkins-hbase6.apache.org,33239,1713093321027 2024-04-14T11:18:08,855 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-04-14T11:18:08,857 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase6.apache.org,33239,1713093321027] 2024-04-14T11:18:08,857 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase6.apache.org,33239,1713093321027; numProcessing=1 2024-04-14T11:18:08,860 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/draining/jenkins-hbase6.apache.org,33239,1713093321027 already deleted, retry=false 2024-04-14T11:18:08,860 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; jenkins-hbase6.apache.org,33239,1713093321027 expired; onlineServers=0 2024-04-14T11:18:08,860 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2554): ***** STOPPING region server 'jenkins-hbase6.apache.org,33819,1713093319730' ***** 2024-04-14T11:18:08,860 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2568): STOPPED: Cluster shutdown set; onlineServer=0 2024-04-14T11:18:08,860 DEBUG [M:0;jenkins-hbase6:33819 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3446082a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=jenkins-hbase6.apache.org/172.31.2.20:0 2024-04-14T11:18:08,860 INFO [M:0;jenkins-hbase6:33819 {}] regionserver.HRegionServer(1221): stopping server jenkins-hbase6.apache.org,33819,1713093319730 2024-04-14T11:18:08,860 INFO [M:0;jenkins-hbase6:33819 {}] regionserver.HRegionServer(1247): stopping server jenkins-hbase6.apache.org,33819,1713093319730; all regions closed. 2024-04-14T11:18:08,860 DEBUG [M:0;jenkins-hbase6:33819 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-04-14T11:18:08,860 DEBUG [M:0;jenkins-hbase6:33819 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-04-14T11:18:08,860 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-04-14T11:18:08,860 DEBUG [M:0;jenkins-hbase6:33819 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-04-14T11:18:08,860 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.small.0-1713093322674 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.small.0-1713093322674,5,FailOnTimeoutGroup] 2024-04-14T11:18:08,860 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.large.0-1713093322673 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.large.0-1713093322673,5,FailOnTimeoutGroup] 2024-04-14T11:18:08,861 INFO [M:0;jenkins-hbase6:33819 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase6:0 had [] on shutdown 2024-04-14T11:18:08,861 DEBUG [M:0;jenkins-hbase6:33819 {}] master.HMaster(1733): Stopping service threads 2024-04-14T11:18:08,862 INFO [M:0;jenkins-hbase6:33819 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-04-14T11:18:08,862 ERROR [M:0;jenkins-hbase6:33819 {}] procedure2.ProcedureExecutor(702): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (444186969) connection to localhost/127.0.0.1:40697 from jenkins,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-04-14T11:18:08,862 INFO [M:0;jenkins-hbase6:33819 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-04-14T11:18:08,863 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-04-14T11:18:08,863 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-04-14T11:18:08,863 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-04-14T11:18:08,863 DEBUG [M:0;jenkins-hbase6:33819 {}] zookeeper.ZKUtil(347): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-04-14T11:18:08,863 WARN [M:0;jenkins-hbase6:33819 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-04-14T11:18:08,863 INFO [M:0;jenkins-hbase6:33819 {}] assignment.AssignmentManager(389): Stopping assignment manager 2024-04-14T11:18:08,863 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-04-14T11:18:08,863 INFO [M:0;jenkins-hbase6:33819 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-04-14T11:18:08,864 DEBUG [M:0;jenkins-hbase6:33819 {}] regionserver.HRegion(1676): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-04-14T11:18:08,864 INFO [M:0;jenkins-hbase6:33819 {}] regionserver.HRegion(1698): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-04-14T11:18:08,864 DEBUG [M:0;jenkins-hbase6:33819 {}] regionserver.HRegion(1719): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-04-14T11:18:08,864 DEBUG [M:0;jenkins-hbase6:33819 {}] regionserver.HRegion(1786): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-04-14T11:18:08,864 DEBUG [M:0;jenkins-hbase6:33819 {}] regionserver.HRegion(1796): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-04-14T11:18:08,864 INFO [M:0;jenkins-hbase6:33819 {}] regionserver.HRegion(2832): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=809.04 KB heapSize=996.57 KB 2024-04-14T11:18:08,878 DEBUG [M:0;jenkins-hbase6:33819 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d5dc0423fa9a421880d15a73728ac727 is 88, key is hbase:meta,,1/info:sn/1713093323148/Put/seqid=0 2024-04-14T11:18:08,958 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-04-14T11:18:08,958 INFO [RS:0;jenkins-hbase6:33239 {}] regionserver.HRegionServer(1304): Exiting; stopping=jenkins-hbase6.apache.org,33239,1713093321027; zookeeper connection closed. 2024-04-14T11:18:08,958 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): regionserver:33239-0x1004d666d980001, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-04-14T11:18:08,958 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3ceca085 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3ceca085 2024-04-14T11:18:08,958 INFO [Listener at localhost/34025 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-04-14T11:18:09,283 INFO [M:0;jenkins-hbase6:33819 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=2309 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d5dc0423fa9a421880d15a73728ac727 2024-04-14T11:18:09,303 DEBUG [M:0;jenkins-hbase6:33819 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fb5788201b3044ae99dc5aa71dcea8e2 is 2277, key is \x00\x00\x00\x00\x00\x00\x00\xA7/proc:d/1713093461446/Put/seqid=0 2024-04-14T11:18:09,309 INFO [M:0;jenkins-hbase6:33819 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=808.44 KB at sequenceid=2309 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fb5788201b3044ae99dc5aa71dcea8e2 2024-04-14T11:18:09,313 INFO [M:0;jenkins-hbase6:33819 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fb5788201b3044ae99dc5aa71dcea8e2 2024-04-14T11:18:09,326 DEBUG [M:0;jenkins-hbase6:33819 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/271599611571439ba7e7696ec858fda5 is 82, key is jenkins-hbase6.apache.org,33239,1713093321027/rs:state/1713093322769/Put/seqid=0 2024-04-14T11:18:09,330 INFO [M:0;jenkins-hbase6:33819 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=2309 (bloomFilter=true), to=hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/271599611571439ba7e7696ec858fda5 2024-04-14T11:18:09,335 DEBUG [M:0;jenkins-hbase6:33819 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d5dc0423fa9a421880d15a73728ac727 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d5dc0423fa9a421880d15a73728ac727 2024-04-14T11:18:09,340 INFO [M:0;jenkins-hbase6:33819 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d5dc0423fa9a421880d15a73728ac727, entries=8, sequenceid=2309, filesize=5.5 K 2024-04-14T11:18:09,341 DEBUG [M:0;jenkins-hbase6:33819 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fb5788201b3044ae99dc5aa71dcea8e2 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fb5788201b3044ae99dc5aa71dcea8e2 2024-04-14T11:18:09,345 INFO [M:0;jenkins-hbase6:33819 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fb5788201b3044ae99dc5aa71dcea8e2 2024-04-14T11:18:09,345 INFO [M:0;jenkins-hbase6:33819 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fb5788201b3044ae99dc5aa71dcea8e2, entries=199, sequenceid=2309, filesize=48.2 K 2024-04-14T11:18:09,346 DEBUG [M:0;jenkins-hbase6:33819 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/271599611571439ba7e7696ec858fda5 as hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/271599611571439ba7e7696ec858fda5 2024-04-14T11:18:09,351 INFO [M:0;jenkins-hbase6:33819 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40697/user/jenkins/test-data/701eedd0-7897-04ac-0603-e5147733047a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/271599611571439ba7e7696ec858fda5, entries=1, sequenceid=2309, filesize=5.1 K 2024-04-14T11:18:09,351 INFO [M:0;jenkins-hbase6:33819 {}] regionserver.HRegion(3035): Finished flush of dataSize ~809.04 KB/828452, heapSize ~996.27 KB/1020184, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 487ms, sequenceid=2309, compaction requested=false 2024-04-14T11:18:09,352 INFO [M:0;jenkins-hbase6:33819 {}] regionserver.HRegion(1917): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-04-14T11:18:09,353 DEBUG [M:0;jenkins-hbase6:33819 {}] regionserver.HRegion(1630): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-04-14T11:18:09,356 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-04-14T11:18:09,356 INFO [M:0;jenkins-hbase6:33819 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2024-04-14T11:18:09,357 INFO [M:0;jenkins-hbase6:33819 {}] ipc.NettyRpcServer(352): Stopping server on /172.31.2.20:33819 2024-04-14T11:18:09,359 DEBUG [M:0;jenkins-hbase6:33819 {}] zookeeper.RecoverableZooKeeper(224): Node /hbase/rs/jenkins-hbase6.apache.org,33819,1713093319730 already deleted, retry=false 2024-04-14T11:18:09,461 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-04-14T11:18:09,461 INFO [M:0;jenkins-hbase6:33819 {}] regionserver.HRegionServer(1304): Exiting; stopping=jenkins-hbase6.apache.org,33819,1713093319730; zookeeper connection closed. 2024-04-14T11:18:09,461 DEBUG [Listener at localhost/34025-EventThread {}] zookeeper.ZKWatcher(604): master:33819-0x1004d666d980000, quorum=127.0.0.1:60913, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-04-14T11:18:09,463 WARN [Listener at localhost/34025 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2024-04-14T11:18:09,468 INFO [Listener at localhost/34025 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2024-04-14T11:18:09,572 WARN [BP-1613181115-172.31.2.20-1713093316581 heartbeating to localhost/127.0.0.1:40697 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-04-14T11:18:09,573 WARN [BP-1613181115-172.31.2.20-1713093316581 heartbeating to localhost/127.0.0.1:40697 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1613181115-172.31.2.20-1713093316581 (Datanode Uuid 5f4d4e3a-51b2-4c7c-b5ce-4641021b8767) service to localhost/127.0.0.1:40697 2024-04-14T11:18:09,577 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/cluster_4b1b29d7-47f8-fcef-3a29-5ae20b7f3813/dfs/data/data1/current/BP-1613181115-172.31.2.20-1713093316581 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-04-14T11:18:09,577 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/9aa5ebc4-e9ba-c6ed-1d8b-1711f26581c4/cluster_4b1b29d7-47f8-fcef-3a29-5ae20b7f3813/dfs/data/data2/current/BP-1613181115-172.31.2.20-1713093316581 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-04-14T11:18:09,613 INFO [Listener at localhost/34025 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2024-04-14T11:18:09,801 INFO [Listener at localhost/34025 {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-04-14T11:18:10,014 INFO [Listener at localhost/34025 {}] hbase.HBaseTestingUtility(1347): Minicluster is down